From b0bc3616ef16c7a8eb1284cb9b9a6dbba57e2784 Mon Sep 17 00:00:00 2001 From: Stephanie Reeder Date: Thu, 11 Jun 2015 12:40:52 -0600 Subject: [PATCH 01/15] remove services and models from odmtools, begin to use api --- odmtools/odmdata/__init__.py | 12 +- odmtools/odmdata/base.py | 2 - odmtools/odmdata/censor_code_cv.py | 11 - odmtools/odmdata/data_type_cv.py | 11 - odmtools/odmdata/data_value.py | 74 -- odmtools/odmdata/general_category_cv.py | 11 - odmtools/odmdata/iso_metadata.py | 15 - odmtools/odmdata/lab_method.py | 15 - odmtools/odmdata/method.py | 12 - odmtools/odmdata/odm_version.py | 10 - odmtools/odmdata/offset_type.py | 17 - odmtools/odmdata/qualifier.py | 12 - odmtools/odmdata/quality_control_level.py | 13 - odmtools/odmdata/sample.py | 18 - odmtools/odmdata/sample_medium_cv.py | 11 - odmtools/odmdata/sample_type_cv.py | 11 - odmtools/odmdata/series.py | 129 ---- odmtools/odmdata/session_factory.py | 57 -- odmtools/odmdata/site.py | 37 - odmtools/odmdata/site_type_cv.py | 11 - odmtools/odmdata/source.py | 27 - odmtools/odmdata/spatial_reference.py | 14 - odmtools/odmdata/speciation_cv.py | 11 - odmtools/odmdata/topic_category_cv.py | 11 - odmtools/odmdata/unit.py | 13 - odmtools/odmdata/value_type_cv.py | 11 - odmtools/odmdata/variable.py | 28 - odmtools/odmdata/variable_name_cv.py | 11 - odmtools/odmdata/vertical_datum_cv.py | 11 - odmtools/odmservices/__init__.py | 18 - odmtools/odmservices/cv_service.py | 100 --- odmtools/odmservices/edit_service.py | 583 ---------------- odmtools/odmservices/export_service.py | 401 ----------- odmtools/odmservices/series_service.py | 782 ---------------------- odmtools/odmservices/service_manager.py | 216 ------ 35 files changed, 5 insertions(+), 2721 deletions(-) delete mode 100644 odmtools/odmdata/base.py delete mode 100644 odmtools/odmdata/censor_code_cv.py delete mode 100644 odmtools/odmdata/data_type_cv.py delete mode 100644 odmtools/odmdata/data_value.py delete mode 100644 odmtools/odmdata/general_category_cv.py delete mode 100644 odmtools/odmdata/iso_metadata.py delete mode 100644 odmtools/odmdata/lab_method.py delete mode 100644 odmtools/odmdata/method.py delete mode 100644 odmtools/odmdata/odm_version.py delete mode 100644 odmtools/odmdata/offset_type.py delete mode 100644 odmtools/odmdata/qualifier.py delete mode 100644 odmtools/odmdata/quality_control_level.py delete mode 100644 odmtools/odmdata/sample.py delete mode 100644 odmtools/odmdata/sample_medium_cv.py delete mode 100644 odmtools/odmdata/sample_type_cv.py delete mode 100644 odmtools/odmdata/series.py delete mode 100644 odmtools/odmdata/session_factory.py delete mode 100644 odmtools/odmdata/site.py delete mode 100644 odmtools/odmdata/site_type_cv.py delete mode 100644 odmtools/odmdata/source.py delete mode 100644 odmtools/odmdata/spatial_reference.py delete mode 100644 odmtools/odmdata/speciation_cv.py delete mode 100644 odmtools/odmdata/topic_category_cv.py delete mode 100644 odmtools/odmdata/unit.py delete mode 100644 odmtools/odmdata/value_type_cv.py delete mode 100644 odmtools/odmdata/variable.py delete mode 100644 odmtools/odmdata/variable_name_cv.py delete mode 100644 odmtools/odmdata/vertical_datum_cv.py delete mode 100644 odmtools/odmservices/__init__.py delete mode 100644 odmtools/odmservices/cv_service.py delete mode 100644 odmtools/odmservices/edit_service.py delete mode 100644 odmtools/odmservices/export_service.py delete mode 100644 odmtools/odmservices/series_service.py delete mode 100755 odmtools/odmservices/service_manager.py diff --git a/odmtools/odmdata/__init__.py b/odmtools/odmdata/__init__.py index aaab1ef..9e7c314 100644 --- a/odmtools/odmdata/__init__.py +++ b/odmtools/odmdata/__init__.py @@ -1,3 +1,4 @@ +''' from base import Base from censor_code_cv import CensorCodeCV from data_type_cv import DataTypeCV @@ -26,7 +27,6 @@ from variable import Variable from variable_name_cv import VariableNameCV from vertical_datum_cv import VerticalDatumCV -from memory_database import MemoryDatabase from series import copy_series from data_value import copy_data_value @@ -64,11 +64,9 @@ 'copy_series', 'copy_data_value' ] +''' -# pyinstaller needs pymysql, psycopg2, and pyodbc (If on windows) import sys -if sys.platform is not 'darwin': - import pymysql - import psycopg2 - import pyodbc - +sys.path.append('/Users/stephanie/DEV/ODM2PythonAPI/') +import src.api as api +from src.api.ODMconnection import SessionFactory \ No newline at end of file diff --git a/odmtools/odmdata/base.py b/odmtools/odmdata/base.py deleted file mode 100644 index c1da040..0000000 --- a/odmtools/odmdata/base.py +++ /dev/null @@ -1,2 +0,0 @@ -from sqlalchemy.ext.declarative import declarative_base -Base = declarative_base() \ No newline at end of file diff --git a/odmtools/odmdata/censor_code_cv.py b/odmtools/odmdata/censor_code_cv.py deleted file mode 100644 index 254a825..0000000 --- a/odmtools/odmdata/censor_code_cv.py +++ /dev/null @@ -1,11 +0,0 @@ -from sqlalchemy import Column, String -from base import Base - -class CensorCodeCV(Base): - __tablename__ = 'CensorCodeCV' - - term = Column('Term', String, primary_key=True) - definition = Column('Definition', String) - - def __repr__(self): - return "" % (self.term, self.definition) \ No newline at end of file diff --git a/odmtools/odmdata/data_type_cv.py b/odmtools/odmdata/data_type_cv.py deleted file mode 100644 index da23bfe..0000000 --- a/odmtools/odmdata/data_type_cv.py +++ /dev/null @@ -1,11 +0,0 @@ -from sqlalchemy import Column, String -from base import Base - -class DataTypeCV(Base): - __tablename__ = 'DataTypeCV' - - term = Column('Term', String, primary_key=True) - definition = Column('Definition', String) - - def __repr__(self): - return "" % (self.term, self.definition) \ No newline at end of file diff --git a/odmtools/odmdata/data_value.py b/odmtools/odmdata/data_value.py deleted file mode 100644 index 687414a..0000000 --- a/odmtools/odmdata/data_value.py +++ /dev/null @@ -1,74 +0,0 @@ -# Declare a mapped class -from sqlalchemy import Column, Integer, Float, DateTime, ForeignKey, String -from sqlalchemy.orm import relationship -from base import Base -from site import Site -from variable import Variable -from qualifier import Qualifier -from method import Method -from source import Source -from quality_control_level import QualityControlLevel -from offset_type import OffsetType -from sample import Sample - -def copy_data_value(from_dv): - new = DataValue() - new.data_value = from_dv.data_value - new.value_accuracy = from_dv.value_accuracy - new.local_date_time = from_dv.local_date_time - new.utc_offset = from_dv.utc_offset - new.date_time_utc = from_dv.date_time_utc - new.site_id = from_dv.site_id - new.variable_id = from_dv.variable_id - new.offset_value = from_dv.offset_value - new.offset_type_id = from_dv.offset_type_id - new.censor_code = from_dv.censor_code - new.qualifier_id = from_dv.qualifier_id - new.method_id = from_dv.method_id - new.source_id = from_dv.source_id - new.sample_id = from_dv.sample_id - new.derived_from_id = from_dv.derived_from_id - new.quality_control_level_id = from_dv.quality_control_level_id - return new - -class DataValue(Base): - __tablename__ = 'DataValues' - - id = Column('ValueID', Integer, primary_key=True) - data_value = Column('DataValue', Float) - value_accuracy = Column('ValueAccuracy', Float) - local_date_time = Column('LocalDateTime', DateTime) - utc_offset = Column('UTCOffset', Float) - date_time_utc = Column('DateTimeUTC', DateTime) - site_id = Column('SiteID', Integer, ForeignKey('Sites.SiteID'), nullable=False) - variable_id = Column('VariableID', Integer, ForeignKey('Variables.VariableID'), nullable=False) - offset_value = Column('OffsetValue', Float) - offset_type_id = Column('OffsetTypeID', Integer, ForeignKey('OffsetTypes.OffsetTypeID')) - censor_code = Column('CensorCode', String) - qualifier_id = Column('QualifierID', Integer, ForeignKey('Qualifiers.QualifierID')) - method_id = Column('MethodID', Integer, ForeignKey('Methods.MethodID'), nullable=False) - source_id = Column('SourceID', Integer, ForeignKey('Sources.SourceID'), nullable=False) - sample_id = Column('SampleID', Integer, ForeignKey('Samples.SampleID')) - derived_from_id = Column('DerivedFromID', Integer) - quality_control_level_id = Column('QualityControlLevelID', Integer, ForeignKey('QualityControlLevels.QualityControlLevelID'), nullable=False) - - # relationships - site = relationship(Site) - variable = relationship(Variable) - method = relationship(Method) - source = relationship(Source) - quality_control_level = relationship(QualityControlLevel) - - qualifier = relationship(Qualifier) - offset_type = relationship(OffsetType) - sample = relationship(Sample) - - def list_repr(self): - return [self.id, self.data_value, self.value_accuracy, self.local_date_time, - self.utc_offset, self.date_time_utc, self.site_id, self.variable_id, - self.offset_value, self.offset_type_id, self.censor_code, self.qualifier_id, - self.method_id, self.source_id, self.sample_id, self.derived_from_id, - self.quality_control_level_id] - - def __repr__(self): - return "" % (self.data_value, self.local_date_time, self.value_accuracy) diff --git a/odmtools/odmdata/general_category_cv.py b/odmtools/odmdata/general_category_cv.py deleted file mode 100644 index e721166..0000000 --- a/odmtools/odmdata/general_category_cv.py +++ /dev/null @@ -1,11 +0,0 @@ -from sqlalchemy import Column, String -from base import Base - -class GeneralCategoryCV(Base): - __tablename__ = 'GeneralCategoryCV' - - term = Column('Term', String, primary_key=True) - definition = Column('Definition', String) - - def __repr__(self): - return "" % (self.term, self.definition) \ No newline at end of file diff --git a/odmtools/odmdata/iso_metadata.py b/odmtools/odmdata/iso_metadata.py deleted file mode 100644 index cd090d4..0000000 --- a/odmtools/odmdata/iso_metadata.py +++ /dev/null @@ -1,15 +0,0 @@ -from sqlalchemy import Column, Integer, String -from base import Base - -class ISOMetadata(Base): - __tablename__ = 'ISOMetadata' - - id = Column('MetadataID', Integer, primary_key=True) - topic_category = Column('TopicCategory', String, nullable=False) - title = Column('Title', String, nullable=False) - abstract = Column('Abstract', String, nullable=False) - profile_version = Column('ProfileVersion', String, nullable=False) - metadata_link = Column('MetadataLink', String) - - def __repr__(self): - return "" % (self.id, self.topic_category, self.title) \ No newline at end of file diff --git a/odmtools/odmdata/lab_method.py b/odmtools/odmdata/lab_method.py deleted file mode 100644 index 1b933cb..0000000 --- a/odmtools/odmdata/lab_method.py +++ /dev/null @@ -1,15 +0,0 @@ -from sqlalchemy import Column, String, Integer -from base import Base - -class LabMethod(Base): - __tablename__ = 'LabMethods' - - id = Column('LabMethodID', Integer, primary_key=True) - name = Column('LabName', String, nullable=False) - organization = Column('LabOrganization', String, nullable=False) - method_name = Column('LabMethodName', String, nullable=False) - method_description = Column('LabMethodDescription', String, nullable=False) - method_link = Column('LabMethodLink', String) - - def __repr__(self): - return "" % (self.id, self.name, self.organization, self.method_name) \ No newline at end of file diff --git a/odmtools/odmdata/method.py b/odmtools/odmdata/method.py deleted file mode 100644 index f3b9d8f..0000000 --- a/odmtools/odmdata/method.py +++ /dev/null @@ -1,12 +0,0 @@ -from sqlalchemy import Column, Integer, String -from base import Base - -class Method(Base): - __tablename__ = 'Methods' - - id = Column('MethodID', Integer, primary_key=True) - description = Column('MethodDescription', String, nullable=False) - link = Column('MethodLink', String) - - def __repr__(self): - return "" % (self.id, self.description, self.link) \ No newline at end of file diff --git a/odmtools/odmdata/odm_version.py b/odmtools/odmdata/odm_version.py deleted file mode 100644 index 3fe4420..0000000 --- a/odmtools/odmdata/odm_version.py +++ /dev/null @@ -1,10 +0,0 @@ -from sqlalchemy import String, Column -from base import Base - -class ODMVersion(Base): - __tablename__ = 'ODMVersion' - - version_number = Column('VersionNumber', String, primary_key=True) - - def __repr__(self): - return "" % (self.version_number) \ No newline at end of file diff --git a/odmtools/odmdata/offset_type.py b/odmtools/odmdata/offset_type.py deleted file mode 100644 index f1fc6ca..0000000 --- a/odmtools/odmdata/offset_type.py +++ /dev/null @@ -1,17 +0,0 @@ -from sqlalchemy import Column, Integer, ForeignKey, String -from sqlalchemy.orm import relationship -from base import Base -from unit import Unit - -class OffsetType(Base): - __tablename__ = 'OffsetTypes' - - id = Column('OffsetTypeID', Integer, primary_key=True) - unit_id = Column('OffsetUnitsID', Integer, ForeignKey('Units.UnitsID'), nullable=False) - description = Column('OffsetDescription', String) - - # relationships - unit = relationship(Unit) - - def __repr__(self): - return "" % (self.id, self.unit_id, self.description) \ No newline at end of file diff --git a/odmtools/odmdata/qualifier.py b/odmtools/odmdata/qualifier.py deleted file mode 100644 index 052febb..0000000 --- a/odmtools/odmdata/qualifier.py +++ /dev/null @@ -1,12 +0,0 @@ -from sqlalchemy import Column, String, Integer -from base import Base - -class Qualifier(Base): - __tablename__ = 'Qualifiers' - - id = Column('QualifierID', Integer, primary_key=True) - code = Column('QualifierCode', String, nullable=False) - description = Column('QualifierDescription', String, nullable=False) - - def __repr__(self): - return "" % (self.id, self.code, self.description) \ No newline at end of file diff --git a/odmtools/odmdata/quality_control_level.py b/odmtools/odmdata/quality_control_level.py deleted file mode 100644 index 5e3c226..0000000 --- a/odmtools/odmdata/quality_control_level.py +++ /dev/null @@ -1,13 +0,0 @@ -from sqlalchemy import Column, String, Integer -from base import Base - -class QualityControlLevel(Base): - __tablename__ = 'QualityControlLevels' - - id = Column('QualityControlLevelID', Integer, primary_key=True) - code = Column('QualityControlLevelCode', String, nullable=False) - definition = Column('Definition', String, nullable=False) - explanation = Column('Explanation', String, nullable=False) - - def __repr__(self): - return "" % (self.id, self.code, self.definition, self.explanation) \ No newline at end of file diff --git a/odmtools/odmdata/sample.py b/odmtools/odmdata/sample.py deleted file mode 100644 index 9d18b33..0000000 --- a/odmtools/odmdata/sample.py +++ /dev/null @@ -1,18 +0,0 @@ -from sqlalchemy import Column, Integer, String, ForeignKey -from sqlalchemy.orm import relationship -from base import Base -from lab_method import LabMethod - -class Sample(Base): - __tablename__ = 'Samples' - - id = Column('SampleID', Integer, primary_key=True) - type = Column('SampleType', String, nullable=False) - lab_sample_code = Column('LabSampleCode', String, nullable=False) - lab_method_id = Column('LabMethodID', Integer, ForeignKey('LabMethods.LabMethodID'), nullable=False) - - # relationships - lab_method = relationship(LabMethod) - - def __repr__(self): - return "" % (self.id, self.type, self.lab_sample_code, self.lab_method_id) \ No newline at end of file diff --git a/odmtools/odmdata/sample_medium_cv.py b/odmtools/odmdata/sample_medium_cv.py deleted file mode 100644 index 277f2bd..0000000 --- a/odmtools/odmdata/sample_medium_cv.py +++ /dev/null @@ -1,11 +0,0 @@ -from sqlalchemy import Column, String -from base import Base - -class SampleMediumCV(Base): - __tablename__ = 'SampleMediumCV' - - term = Column('Term', String, primary_key=True) - definition = Column('Definition', String) - - def __repr__(self): - return "" % (self.term, self.definition) \ No newline at end of file diff --git a/odmtools/odmdata/sample_type_cv.py b/odmtools/odmdata/sample_type_cv.py deleted file mode 100644 index caaf4c0..0000000 --- a/odmtools/odmdata/sample_type_cv.py +++ /dev/null @@ -1,11 +0,0 @@ -from sqlalchemy import Column, String -from base import Base - -class SampleTypeCV(Base): - __tablename__ = 'SampleTypeCV' - - term = Column('Term', String, primary_key=True) - definition = Column('Definition', String) - - def __repr__(self): - return "" % (self.term, self.definition) \ No newline at end of file diff --git a/odmtools/odmdata/series.py b/odmtools/odmdata/series.py deleted file mode 100644 index 3916544..0000000 --- a/odmtools/odmdata/series.py +++ /dev/null @@ -1,129 +0,0 @@ -from collections import OrderedDict # Requires Python 2.7 >= - -from sqlalchemy import Column, Integer, ForeignKey, String, Float, DateTime -from sqlalchemy.orm import relationship - -from base import Base -from site import Site -from variable import Variable -from method import Method -from source import Source -from quality_control_level import QualityControlLevel - -def copy_series(from_series): - new = Series() - new.site_id = from_series.site_id - new.site_code = from_series.site_code - new.site_name = from_series.site_name - new.variable_id = from_series.variable_id - new.variable_code = from_series.variable_code - new.variable_name = from_series.variable_name - new.speciation = from_series.speciation - new.variable_units_id = from_series.variable_units_id - new.variable_units_name = from_series.variable_units_name - new.sample_medium = from_series.sample_medium - new.value_type = from_series.value_type - new.time_support = from_series.time_support - new.time_units_id = from_series.time_units_id - new.time_units_name = from_series.time_units_name - new.data_type = from_series.data_type - new.general_category = from_series.general_category - new.method_id = from_series.method_id - new.method_description = from_series.method_description - new.source_id = from_series.source_id - new.source_description = from_series.source_description - new.organization = from_series.organization - new.citation = from_series.citation - new.quality_control_level_id = from_series.quality_control_level_id - new.quality_control_level_code = from_series.quality_control_level_code - new.begin_date_time = from_series.begin_date_time - new.begin_date_time_utc = from_series.begin_date_time_utc - new.end_date_time_utc = from_series.end_date_time_utc - new.value_count = from_series.value_count - return new -class Series(Base): - __tablename__ = 'seriescatalog' - - id = Column('SeriesID', Integer, primary_key=True) - site_id = Column('SiteID', Integer, ForeignKey('Sites.SiteID'), nullable=False) - site_code = Column('SiteCode', String) - site_name = Column('SiteName', String) - variable_id = Column('VariableID', Integer, ForeignKey('Variables.VariableID'), nullable=False) - variable_code = Column('VariableCode', String) - variable_name = Column('VariableName', String) - speciation = Column('Speciation', String) - variable_units_id = Column('VariableUnitsID', Integer) - variable_units_name = Column('VariableUnitsName', String) - sample_medium = Column('SampleMedium', String) - value_type = Column('ValueType', String) - time_support = Column('TimeSupport', Float) - time_units_id = Column('TimeUnitsID', Integer) - time_units_name = Column('TimeUnitsName', String) - data_type = Column('DataType', String) - general_category = Column('GeneralCategory', String) - method_id = Column('MethodID', Integer, ForeignKey('Methods.MethodID'), nullable=False) - method_description = Column('MethodDescription', String) - source_id = Column('SourceID', Integer, ForeignKey('Sources.SourceID'), nullable=False) - source_description = Column('SourceDescription', String) - organization = Column('Organization', String) - citation = Column('Citation', String) - quality_control_level_id = Column('QualityControlLevelID', Integer, - ForeignKey('QualityControlLevels.QualityControlLevelID'), nullable=False) - quality_control_level_code = Column('QualityControlLevelCode', String) - begin_date_time = Column('BeginDateTime', DateTime) - end_date_time = Column('EndDateTime', DateTime) - begin_date_time_utc = Column('BeginDateTimeUTC', DateTime) - end_date_time_utc = Column('EndDateTimeUTC', DateTime) - value_count = Column('ValueCount', Integer) - - data_values = relationship("DataValue", - primaryjoin="and_(DataValue.site_id == Series.site_id, " - "DataValue.variable_id == Series.variable_id, " - "DataValue.method_id == Series.method_id, " - "DataValue.source_id == Series.source_id, " - "DataValue.quality_control_level_id == Series.quality_control_level_id)", - foreign_keys="[DataValue.site_id, DataValue.variable_id, DataValue.method_id, DataValue.source_id, DataValue.quality_control_level_id]", - order_by="DataValue.local_date_time", - backref="series") - - site = relationship(Site) - variable = relationship(Variable) - method = relationship(Method) - source = relationship(Source) - quality_control_level = relationship(QualityControlLevel) - - # TODO add all to repr - def __repr__(self): - return "" % (self.id, self.site_name, self.variable_code, self.variable_name) - - - def get_table_columns(self): - return self.__table__.columns.keys() - - def list_repr(self): - return [self.id, self.site_id, self.site_code, self.site_name, self.variable_id, self.variable_code, - self.variable_name, self.speciation, self.variable_units_id, self.variable_units_name, - self.sample_medium, self.value_type, self.time_support, self.time_units_id, self.time_units_name, - self.data_type, self.general_category, self.method_id, self.method_description, - self.source_id, self.source_description, self.organization, self.citation, - self.quality_control_level_id, self.quality_control_level_code, self.begin_date_time, - self.end_date_time, self.begin_date_time_utc, self.end_date_time_utc, self.value_count] - -def returnDict(): - keys = ['SeriesID', 'SiteID', 'SiteCode', 'SiteName', 'VariableID', 'VariableCode', 'VariableName', 'Speciation', - 'VariableUnitsID', 'VariableUnitsName', 'SampleMedium', 'ValueType', 'TimeSupport', 'TimeUnitsID', - 'TimeUnitsName', 'DataType', 'GeneralCategory', 'MethodID', 'MethodDescription', 'SourceID', - 'SourceDescription', 'Organization', 'Citation', 'QualityControlLevelID', 'QualityControlLevelCode', - 'BeginDateTime', 'EndDateTime', 'BeginDateTimeUTC', 'EndDateTimeUTC', 'ValueCount' - ] - values = ['id', 'site_id', 'site_code', 'site_name', 'variable_id', 'variable_code', 'variable_name', 'speciation', - 'variable_units_id', 'variable_units_name', 'sample_medium', 'value_type', 'time_support', - 'time_units_id', 'time_units_name', 'data_type', 'general_category', 'method_id', 'method_description', - 'source_id', 'source_description', 'organization', 'citation', 'quality_control_level_id', - 'quality_control_level_code', 'begin_date_time', 'end_date_time', 'begin_date_time_utc', - 'end_date_time_utc', 'value_count' - ] - return OrderedDict(zip(keys, values)) - - - diff --git a/odmtools/odmdata/session_factory.py b/odmtools/odmdata/session_factory.py deleted file mode 100644 index 0d2d488..0000000 --- a/odmtools/odmdata/session_factory.py +++ /dev/null @@ -1,57 +0,0 @@ -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - - -class SessionFactory(): - def __init__(self, connection_string, echo): - self.engine = create_engine(connection_string, encoding='utf-8', echo=echo, pool_recycle=3600, - pool_size=20) - self.psql_test_engine = create_engine(connection_string, encoding='utf-8', echo=echo, pool_recycle=3600, - connect_args={'connect_timeout': 1}) - self.ms_test_engine = create_engine(connection_string, encoding='utf-8', echo=echo, pool_recycle=3600, - connect_args={'timeout': 1}) - self.my_test_engine = create_engine(connection_string, encoding='utf-8', echo=echo, pool_recycle=3600, - connect_args={'connect_timeout': 1}) - - ''' - # Removing pool_timeout and max_overflow allowed the tests to pass - self.engine = create_engine(connection_string, encoding='utf-8', echo=echo, pool_recycle=3600, - pool_timeout=5, pool_size=20, max_overflow=0) - self.psql_test_engine = create_engine(connection_string, encoding='utf-8', echo=echo, pool_recycle=3600, - pool_timeout=5, max_overflow=0, connect_args={'connect_timeout': 1}) - self.ms_test_engine = create_engine(connection_string, encoding='utf-8', echo=echo, pool_recycle=3600, - pool_timeout=5, max_overflow=0, connect_args={'timeout': 1}) - self.my_test_engine = create_engine(connection_string, encoding='utf-8', echo=echo, pool_recycle=3600, - pool_timeout=5, max_overflow=0, connect_args={'connect_timeout': 1}) - ''' - - ''' - # Old code - class SessionFactory(): - def __init__(self, connection_string, echo): - self.engine = create_engine(connection_string, encoding='utf-8', echo=echo, - #pool_size=20, - pool_recycle=3600) - - # Create session maker - self.Session = sessionmaker(bind=self.engine) - - def get_session(self): - return self.Session() - - def __repr__(self): - return "" % (self.engine) - ''' - - # Create session maker - self.Session = sessionmaker(bind=self.engine) - self.psql_test_Session = sessionmaker(bind=self.psql_test_engine) - self.ms_test_Session = sessionmaker(bind=self.ms_test_engine) - self.my_test_Session = sessionmaker(bind=self.my_test_engine) - - def get_session(self): - return self.Session() - - def __repr__(self): - return "" % (self.engine) - diff --git a/odmtools/odmdata/site.py b/odmtools/odmdata/site.py deleted file mode 100644 index c92998a..0000000 --- a/odmtools/odmdata/site.py +++ /dev/null @@ -1,37 +0,0 @@ -# Declare a mapped class -from sqlalchemy import Column, Integer, String, Float, ForeignKey -from sqlalchemy.orm import relationship -from base import Base -from spatial_reference import SpatialReference - -class Site(Base): - __tablename__ = 'Sites' - - id = Column('SiteID', Integer, primary_key=True) - code = Column('SiteCode', String) - name = Column('SiteName', String) - latitude = Column('Latitude', Float) - longitude = Column('Longitude', Float) - lat_long_datum_id = Column('LatLongDatumID', Integer, ForeignKey('SpatialReferences.SpatialReferenceID')) - elevation_m = Column('Elevation_m', Float) - vertical_datum_id = Column('VerticalDatum', Integer) - local_x = Column('LocalX', Float) - local_y = Column('LocalY', Float) - local_projection_id = Column('LocalProjectionID', Integer, ForeignKey('SpatialReferences.SpatialReferenceID')) - pos_accuracy_m = Column('PosAccuracy_m', Float) - state = Column('State', String) - county = Column('County', String) - comments = Column('Comments', String) - - type = Column('SiteType', String) - - # relationships - spatial_ref = relationship(SpatialReference, primaryjoin=("SpatialReference.id==Site.lat_long_datum_id")) - local_spatial_ref = relationship(SpatialReference, primaryjoin=("SpatialReference.id==Site.local_projection_id")) - - def __init__(self, site_code, site_name): - self.code = site_code - self.name = site_name - - def __repr__(self): - return "" % (self.code, self.name) \ No newline at end of file diff --git a/odmtools/odmdata/site_type_cv.py b/odmtools/odmdata/site_type_cv.py deleted file mode 100644 index fc41c7e..0000000 --- a/odmtools/odmdata/site_type_cv.py +++ /dev/null @@ -1,11 +0,0 @@ -from sqlalchemy import Column, String -from base import Base - -class SiteTypeCV(Base): - __tablename__ = 'SiteTypeCV' - - term = Column('Term', String, primary_key=True) - definition = Column('Definition', String) - - def __repr__(self): - return "" % (self.term, self.definition) \ No newline at end of file diff --git a/odmtools/odmdata/source.py b/odmtools/odmdata/source.py deleted file mode 100644 index 08452cc..0000000 --- a/odmtools/odmdata/source.py +++ /dev/null @@ -1,27 +0,0 @@ -from sqlalchemy import Column, String, Integer, ForeignKey -from sqlalchemy.orm import relationship -from base import Base -from iso_metadata import ISOMetadata - -class Source(Base): - __tablename__ = 'Sources' - - id = Column('SourceID', Integer, primary_key=True) - organization = Column('Organization', String, nullable=False) - description = Column('SourceDescription', String, nullable=False) - link = Column('SourceLink', String) - contact_name = Column('ContactName', String, nullable=False) - phone = Column('Phone', String, nullable=False) - email = Column('Email', String, nullable=False) - address = Column('Address', String, nullable=False) - city = Column('City', String, nullable=False) - state = Column('State', String, nullable=False) - zip_code = Column('ZipCode', String, nullable=False) - citation = Column('Citation', String, nullable=False) - iso_metadata_id = Column('MetadataID', Integer, ForeignKey('ISOMetadata.MetadataID'), nullable=False) - - # relationships - iso_metadata = relationship(ISOMetadata) - - def __repr__(self): - return "" % (self.id, self.organization, self.description) \ No newline at end of file diff --git a/odmtools/odmdata/spatial_reference.py b/odmtools/odmdata/spatial_reference.py deleted file mode 100644 index 2b63f39..0000000 --- a/odmtools/odmdata/spatial_reference.py +++ /dev/null @@ -1,14 +0,0 @@ -from sqlalchemy import Column, Integer, String, Boolean -from base import Base - -class SpatialReference(Base): - __tablename__ = 'SpatialReferences' - - id = Column('SpatialReferenceID', Integer, primary_key=True) - srs_id = Column('SRSID', Integer) - srs_name = Column('SRSName', String) - is_geographic = Column('IsGeographic', Boolean) - notes = Column('Notes', String) - - def __repr__(self): - return "" % (self.id, self.srs_name) \ No newline at end of file diff --git a/odmtools/odmdata/speciation_cv.py b/odmtools/odmdata/speciation_cv.py deleted file mode 100644 index 7d6c19f..0000000 --- a/odmtools/odmdata/speciation_cv.py +++ /dev/null @@ -1,11 +0,0 @@ -from sqlalchemy import Column, String -from base import Base - -class SpeciationCV(Base): - __tablename__ = 'SpeciationCV' - - term = Column('Term', String, primary_key=True) - definition = Column('Definition', String) - - def __repr__(self): - return "" % (self.term, self.definition) \ No newline at end of file diff --git a/odmtools/odmdata/topic_category_cv.py b/odmtools/odmdata/topic_category_cv.py deleted file mode 100644 index 54f65c5..0000000 --- a/odmtools/odmdata/topic_category_cv.py +++ /dev/null @@ -1,11 +0,0 @@ -from sqlalchemy import Column, String -from base import Base - -class TopicCategoryCV(Base): - __tablename__ = 'TopicCategoryCV' - - term = Column('Term', String, primary_key=True) - definition = Column('Definition', String) - - def __repr__(self): - return "" % (self.term, self.definition) \ No newline at end of file diff --git a/odmtools/odmdata/unit.py b/odmtools/odmdata/unit.py deleted file mode 100644 index 853da31..0000000 --- a/odmtools/odmdata/unit.py +++ /dev/null @@ -1,13 +0,0 @@ -from sqlalchemy import Column, Integer, String, Unicode -from base import Base - -class Unit(Base): - __tablename__ = 'Units' - - id = Column('UnitsID', Integer, primary_key=True) - name = Column('UnitsName', String) - type = Column('UnitsType', String) - abbreviation = Column('UnitsAbbreviation', String)#(convert_unicode=True)) - - def __repr__(self): - return "" % (self.id, self.name, self.type) \ No newline at end of file diff --git a/odmtools/odmdata/value_type_cv.py b/odmtools/odmdata/value_type_cv.py deleted file mode 100644 index 1a19d8a..0000000 --- a/odmtools/odmdata/value_type_cv.py +++ /dev/null @@ -1,11 +0,0 @@ -from sqlalchemy import Column, String -from base import Base - -class ValueTypeCV(Base): - __tablename__ = 'ValueTypeCV' - - term = Column('Term', String, primary_key=True) - definition = Column('Definition', String) - - def __repr__(self): - return "" % (self.term, self.definition) \ No newline at end of file diff --git a/odmtools/odmdata/variable.py b/odmtools/odmdata/variable.py deleted file mode 100644 index 77d8109..0000000 --- a/odmtools/odmdata/variable.py +++ /dev/null @@ -1,28 +0,0 @@ -from sqlalchemy import Column, Integer, String, ForeignKey, Boolean, Float -from sqlalchemy.orm import relationship -from base import Base -from unit import Unit - -class Variable(Base): - __tablename__ = 'Variables' - - id = Column('VariableID', Integer, primary_key=True) - code = Column('VariableCode', String, nullable=False) - name = Column('VariableName', String, nullable=False) - speciation = Column('Speciation', String, nullable=False) - variable_unit_id = Column('VariableUnitsID', Integer, ForeignKey('Units.UnitsID'), nullable=False) - sample_medium = Column('SampleMedium', String, nullable=False) - value_type = Column('ValueType', String, nullable=False) - is_regular = Column('IsRegular', Boolean, nullable=False) - time_support = Column('TimeSupport', Float, nullable=False) - time_unit_id = Column('TimeUnitsID', Integer, ForeignKey('Units.UnitsID'), nullable=False) - data_type = Column('DataType', String, nullable=False) - general_category = Column('GeneralCategory', String, nullable=False) - no_data_value = Column('NoDataValue', Float, nullable=False) - - # relationships - variable_unit = relationship(Unit, primaryjoin=("Unit.id==Variable.variable_unit_id")) # <-- Uses class attribute names, not table column names - time_unit = relationship(Unit, primaryjoin=("Unit.id==Variable.time_unit_id")) - - def __repr__(self): - return "" % (self.id, self.code, self.name) \ No newline at end of file diff --git a/odmtools/odmdata/variable_name_cv.py b/odmtools/odmdata/variable_name_cv.py deleted file mode 100644 index 0b64596..0000000 --- a/odmtools/odmdata/variable_name_cv.py +++ /dev/null @@ -1,11 +0,0 @@ -from sqlalchemy import Column, String -from base import Base - -class VariableNameCV(Base): - __tablename__ = 'VariableNameCV' - - term = Column('Term', String, primary_key=True) - definition = Column('Definition', String) - - def __repr__(self): - return "" % (self.term, self.definition) \ No newline at end of file diff --git a/odmtools/odmdata/vertical_datum_cv.py b/odmtools/odmdata/vertical_datum_cv.py deleted file mode 100644 index b161b29..0000000 --- a/odmtools/odmdata/vertical_datum_cv.py +++ /dev/null @@ -1,11 +0,0 @@ -from sqlalchemy import Column, String -from base import Base - -class VerticalDatumCV(Base): - __tablename__ = 'VerticalDatumCV' - - term = Column('Term', String, primary_key=True) - definition = Column('Definition', String) - - def __repr__(self): - return "" % (self.term, self.definition) \ No newline at end of file diff --git a/odmtools/odmservices/__init__.py b/odmtools/odmservices/__init__.py deleted file mode 100644 index 3952589..0000000 --- a/odmtools/odmservices/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -from service_manager import ServiceManager -from series_service import SeriesService -from cv_service import CVService -from edit_service import EditService -from export_service import ExportService - -# need to explicitly import these for pyinstaller -import pymysql -import pyodbc -import psycopg2 - -__all__ = [ - 'EditService', - 'CVService', - 'SeriesService', - 'ExportService', - 'ServiceManager', -] \ No newline at end of file diff --git a/odmtools/odmservices/cv_service.py b/odmtools/odmservices/cv_service.py deleted file mode 100644 index 0e92b7c..0000000 --- a/odmtools/odmservices/cv_service.py +++ /dev/null @@ -1,100 +0,0 @@ -# CV imports -from odmtools.odmdata import SessionFactory -from odmtools.odmdata import VerticalDatumCV -from odmtools.odmdata import SiteTypeCV -from odmtools.odmdata import VariableNameCV -from odmtools.odmdata import SpeciationCV -from odmtools.odmdata import SampleMediumCV -from odmtools.odmdata import ValueTypeCV -from odmtools.odmdata import DataTypeCV -from odmtools.odmdata import GeneralCategoryCV -from odmtools.odmdata import CensorCodeCV -from odmtools.odmdata import TopicCategoryCV -from odmtools.odmdata import SampleTypeCV -from odmtools.odmdata import OffsetType -from odmtools.odmdata import Sample -from odmtools.odmdata import Qualifier -from odmtools.odmdata import Unit -from sqlalchemy import not_ - - -class CVService(): - # Accepts a string for creating a SessionFactory, default uses odmdata/connection.cfg - def __init__(self, connection_string="", debug=False): - self._session_factory = SessionFactory(connection_string, debug) - self._edit_session = self._session_factory.get_session() - self._debug = debug - - # Controlled Vocabulary get methods - - #return a list of all terms in the cv - def get_vertical_datum_cvs(self): - result = self._edit_session.query(VerticalDatumCV).order_by(VerticalDatumCV.term).all() - return result - - def get_samples(self): - result = self._edit_session.query(Sample).order_by(Sample.lab_sample_code).all() - return result - - - - def get_site_type_cvs(self): - result = self._edit_session.query(SiteTypeCV).order_by(SiteTypeCV.term).all() - return result - - def get_variable_name_cvs(self): - result = self._edit_session.query(VariableNameCV).order_by(VariableNameCV.term).all() - return result - - def get_offset_type_cvs(self): - result = self._edit_session.query(OffsetType).order_by(OffsetType.id).all() - return result - - def get_speciation_cvs(self): - result = self._edit_session.query(SpeciationCV).order_by(SpeciationCV.term).all() - return result - - def get_sample_medium_cvs(self): - result = self._edit_session.query(SampleMediumCV).order_by(SampleMediumCV.term).all() - return result - - def get_value_type_cvs(self): - result = self._edit_session.query(ValueTypeCV).order_by(ValueTypeCV.term).all() - return result - - def get_data_type_cvs(self): - result = self._edit_session.query(DataTypeCV).order_by(DataTypeCV.term).all() - return result - - def get_general_category_cvs(self): - result = self._edit_session.query(GeneralCategoryCV).order_by(GeneralCategoryCV.term).all() - return result - - def get_censor_code_cvs(self): - result = self._edit_session.query(CensorCodeCV).order_by(CensorCodeCV.term).all() - return result - - def get_sample_type_cvs(self): - result = self._edit_session.query(SampleTypeCV).order_by(SampleTypeCV.term).all() - return result - - def get_units(self): - result = self._edit_session.query(Unit).all() - return result - - def get_units_not_uni(self): - result = self._edit_session.query(Unit).filter(not_(Unit.name.contains('angstrom'))).all() - return result - - def get_units_names(self): - result = self._edit_session.query(Unit.name).all() - return result - - # return a single cv - def get_unit_by_name(self, unit_name): - result = self._edit_session.query(Unit).filter_by(name=unit_name).first() - return result - - def get_unit_by_id(self, unit_id): - result = self._edit_session.query(Unit).filter_by(id=unit_id).first() - return result diff --git a/odmtools/odmservices/edit_service.py b/odmtools/odmservices/edit_service.py deleted file mode 100644 index ccc1f76..0000000 --- a/odmtools/odmservices/edit_service.py +++ /dev/null @@ -1,583 +0,0 @@ -import sqlite3 - - -from odmtools.odmdata import DataValue -from series_service import SeriesService - -from odmtools.odmdata import series as series_module - -import pandas as pd -import datetime -import numpy as np - -import logging -from odmtools.common.logger import LoggerTool - -tool = LoggerTool() -logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG) - - -class EditService(): - # Mutual exclusion: cursor, or connection_string - def __init__(self, series_id, connection=None, connection_string="", debug=False): - ''' - - :param series_id: - :param connection: memory database, contains connection to remote database - :param connection_string: connection to remote database - :param debug: - :return: - ''' - - self._series_id = series_id - self._filter_from_selection = False - self._debug = debug - - if connection_string is "" and connection is not None: - self.memDB= connection - #self._series_service = self.memDB.series_service#SeriesService(connection_string, debug) - - elif connection_string is not "" and connection is None: - from odmtools.odmdata import MemoryDatabase - self.memDB= MemoryDatabase()#(series_service) - self.memDB.set_series_service(SeriesService(connection_string, False)) - - - else: - logger.error("must send in either a remote db connection string or a memory database object") - - logger.debug("Initializing Memory Database") - self.memDB.initEditValues(series_id) - logger.debug("Finished Initializing Memory Database") - self._populate_series() - self.reset_filter() - - def get_series_service(self): - return self.memDB.series_service - - def _populate_series(self): - # [(ID, value, datetime), ...] - #self._cursor.execute("SELECT ValueID, DataValue, LocalDateTime FROM DataValues ORDER BY LocalDateTime") - - self._series_points_df = self.memDB.getDataValuesDF() - - - def _test_filter_previous(self): - - ''' - if not self._filter_from_selection: - self.reset_filter() - ''' - - df = None - - if not self._filter_from_selection: - df = self._series_points_df - else: - df = self.filtered_dataframe - - # Ensure that we're not working with an empty dataframe - - if isinstance(df, pd.DataFrame): - if df.empty: - return self._series_points_df - else: - if not df: - return self._series_points_df - - return df - - def datetime2dataframe(self, datetime_list): - """ Converts datetime_list to a pandas Dataframe - - - :param datetime_list: - :return Pandas.DataFrame: - """ - - result = None - - if isinstance(datetime_list, list): - - result = pd.DataFrame(datetime_list, columns=["LocalDateTime"]) - - result.set_index("LocalDateTime", inplace=True) - - return result - - ################### - # Stubs - ################### - def selectPointsStub(self): - """ - :param filtered_dataframe: - :return: - """ - - ## Convert dataframe into list of datetimes - - filtered_dataframe = self.get_filtered_points() - if isinstance(filtered_dataframe, pd.DataFrame): - if not filtered_dataframe.empty: - datetime_list = filtered_dataframe.index.to_pydatetime() - return datetime_list.tolist() - return [] - - ################### - # Filters - ################### - # operator is a character, either '<' or '>' - def filter_value(self, value, ops): - df = self._test_filter_previous() - - if ops == '>': - self.filtered_dataframe = df[df['DataValue'] > value] - - if ops == '<': - self.filtered_dataframe = df[df['DataValue'] < value] - - - def filter_date(self, before, after): - df = self._test_filter_previous() - if before and after: - self.filtered_dataframe = df[(df.index < before) & (df.index > after)] - - # Data Gaps - def data_gaps(self, value, time_period): - df = self._test_filter_previous() - - time_units = { - 'second': 's', - 'minute': 'm', - 'hour': 'h', - 'day': 'D', - 'week': 'W', - 'month': 'M', - 'year': 'Y' - } - - # make a copy of the dataframe in order to modify it to be in the form we need to determine data gaps - copy_df = df - copy_df['datetime'] = df.index - copy_df['dateprev'] = copy_df['datetime'].shift() - - # ensure that 'value' is an integer - if not isinstance(value, int): - value = int(value) - - # create a bool column indicating which rows meet condition - filtered_results = copy_df['datetime'].diff() >= np.timedelta64(value, time_units[time_period]) - - # filter on rows that passed previous condition - copy_df = copy_df[filtered_results] - - # merge values and remove duplicates. this hack allows for both values to be marked when selecting data gaps - newdf = pd.concat([copy_df['datetime'], copy_df['dateprev']], join='inner') - self.filtered_dataframe = df[df.index.isin(newdf.drop_duplicates().dropna())] - - # clean up - del copy_df - del filtered_results - del newdf - - def change_value_threshold(self, value, operator): - - df = self._test_filter_previous() - - # make a copy of the dataframe in order to modify it to be in the form we need to determine data gaps - copy_df = df - copy_df['values'] = df['DataValue'] - copy_df['diff'] = copy_df['values'].shift() - copy_df["diff_date"] = copy_df['LocalDateTime'].shift() - copy_df['change_threshold'] = abs(df['values'] - df['diff']) - - if not isinstance(value, float): - logger.error("Need to have a float") - return - - copy_df['threshold'] = value - - if operator == ">": - copy_df['matches'] = df['change_threshold'] >= copy_df['threshold'] - - if operator == "<": - copy_df['matches'] = df['change_threshold'] <= copy_df['threshold'] - - filtered_df = copy_df[copy_df['matches']] - tmplist = filtered_df['diff_date'].tolist() + filtered_df.index.tolist() - del copy_df - self.filtered_dataframe = df[df.index.isin(tmplist)] - - - def select_points_tf(self, tf_list): - self._filter_list = tf_list - - #def select_points(self, id_list=[], datetime_list=[]): - def select_points(self, id_list=[], dataframe=[]): - #self.reset_filter() - - # This should be either one or the other. If it's both, id is used first. - # If neither are set this function does nothing. - - if len(id_list) > 0: - for i in range(len(self._series_points)): - if self._series_points[i][0] in id_list: - self._filter_list[i] = True - - if isinstance(dataframe, pd.DataFrame): - result = dataframe.index.astype(datetime.datetime) - self.filtered_dataframe = self._series_points_df[self._series_points_df.index.isin(dataframe.index)] - - - def reset_filter(self): - self.filtered_dataframe = None - - def filter_from_previous(self, value): - self._filter_from_selection = value - - def get_toggle(self): - return self._filter_from_selection - - - ################### - # Gets - ################### - def get_series(self): - return self.memDB.series_service.get_series_by_id(self._series_id) - - def get_series_points(self): - # all point in the series - return self._series_points - - def get_series_points_df(self): - """ - :return Pandas DataFrame: - """ - return self._series_points_df - - def get_filtered_points(self): - """ - :return Pandas DataFrame: - """ - if isinstance(self.filtered_dataframe, pd.DataFrame): - if self.filtered_dataframe.empty: - return None - else: - if not self.filtered_dataframe: - return None - if len(self.filtered_dataframe) > 0: - return self.filtered_dataframe - return None - - def get_filtered_dates(self): - return self.filtered_dataframe - - def get_filter_list(self): - # true or false list the length of the entire series. true indicate the point is selected - return self._filter_list - - def get_qcl(self, qcl_id): - return self.memDB.series_service.get_qcl_by_id(qcl_id) - - def get_method(self, method_id): - return self.memDB.series_service.get_method_by_id(method_id) - - def get_variable(self, variable_id): - logger.debug(variable_id) - return self.memDB.series_service.get_variable_by_id(variable_id) - - - ################# - # Edits - ################# - - def change_value(self, value, operator): - filtered_points = self.get_filtered_points() - - ids = filtered_points["ValueID"].astype(int).tolist() - self.memDB.updateValue(ids, operator, float(value)) - self._populate_series() - - ## update filtered_dataframe - self.filtered_dataframe = self._series_points_df[self._series_points_df['ValueID'].isin(ids)] - - def add_points(self, points): - # todo: add the ability to send in multiple datetimes to a single 'point' - - self.memDB.addPoints(points) - - self._populate_series() - self.reset_filter() - - def delete_points(self): - filtered_points = self.get_filtered_points() - if not filtered_points.empty: - values = filtered_points['ValueID'].astype(float).tolist() - - self.memDB.delete(values) - self._populate_series() - self.filtered_dataframe = None - - def interpolate(self): - ''' - In [75]: ser = Series(np.sort(np.random.uniform(size=100))) - # interpolate at new_index - In [76]: new_index = ser.index | Index([49.25, 49.5, 49.75, 50.25, 50.5, 50.75]) - In [77]: interp_s = ser.reindex(new_index).interpolate(method='pchip') - ''' - - tmp_filter_list =self.get_filtered_points() - df = self._series_points_df - issel = df.index.isin(tmp_filter_list.index) - - mdf = df["DataValue"].mask(issel) - mdf.interpolate(method = "time", inplace=True) - tmp_filter_list["DataValue"]=mdf[issel] - ids = tmp_filter_list['ValueID'].tolist() - - #update_list = [(row["DataValue"], row["ValueID"]) for index, row in tmp_filter_list.iterrows()] - update_list = [{"value": row["DataValue"], "id": row["ValueID"]} for index, row in tmp_filter_list.iterrows()] - - self.memDB.update(update_list) - - - self._populate_series() - - self.filtered_dataframe = self._series_points_df[self._series_points_df['ValueID'].isin(ids)] - - def drift_correction(self, gap_width): - if self.isOneGroup(): - tmp_filter_list =self.get_filtered_points() - startdate =tmp_filter_list.index[0] - x_l = (tmp_filter_list.index[-1]-startdate).total_seconds() - - # y_n = y_0 + G(x_i / x_l) - f = lambda row : row["DataValue"]+(gap_width * ((row.name-startdate).total_seconds() / x_l)) - tmp_filter_list["DataValue"]=tmp_filter_list.apply(f, axis = 1) - - update_list = [{"value": row["DataValue"], "id": row["ValueID"]} for index, row in tmp_filter_list.iterrows()] - - ids = tmp_filter_list['ValueID'].tolist() - self.memDB.update(update_list) - - - self._populate_series() - - self.filtered_dataframe = self._series_points_df[self._series_points_df['ValueID'].isin(ids)] - return True - return False - - def isOneGroup(self): - - issel = self._series_points_df.index.isin(self.get_filtered_points().index) - - found_group = False - count = 0 - - for x in issel: - if x: - if not found_group: - found_group=True - count =count+1 - else: - found_group = False - - if count >1: - return False - if count == 1: - return True - - - def flag(self, qualifier_id): - - filtered_points = self.get_filtered_points() - ''' - query = "UPDATE DataValues SET QualifierID = %s WHERE ValueID = ?" % (qualifier_id) - #self._cursor.executemany(query, [(str(x[0]),) for x in filtered_points]) - self._cursor.executemany(query, [(str(x),) for x in filtered_points["ValueID"].astype(int).tolist()]) - ''' - self.memDB.updateFlag(filtered_points["ValueID"].astype(int).tolist(), qualifier_id) - - ################### - # Save/Restore - ################### - - def restore(self): - self.memDB.rollback() - - self._populate_series() - self.reset_filter() - - def updateSeries(self, var=None, method=None, qcl=None, is_new_series=False): - """ - - :param var: - :param method: - :param qcl: - :param is_new_series: - :return: - """ - - var_id = var.id if var is not None else None - method_id = method.id if method is not None else None - qcl_id = qcl.id if qcl is not None else None - #self.memDB.changeSeriesIDs(var_id, method_id, qcl_id) - dvs = self.memDB.getDataValuesDF() - if var_id is not None: - dvs["VariableID"] = var_id - if method_id is not None: - dvs["MethodID"] = method_id - if qcl_id is not None: - dvs["QualityControlLevelID"] = qcl_id - - - - #if is new series remove valueids - #if is_new_series: - dvs["ValueID"] = None - ''' - for dv in dvs: - dv.id = None - ''' - - series = self.memDB.series_service.get_series_by_id(self._series_id) - logger.debug("original editing series id: %s" % str(series.id)) - - if (var or method or qcl ): - tseries = self.memDB.series_service.get_series_by_id_quint(site_id=int(series.site_id), - var_id=var_id if var else int(series.variable_id), - method_id=method_id if method else int( - series.method_id), - source_id=series.source_id, - qcl_id=qcl_id if qcl else int( - series.quality_control_level_id)) - if tseries: - logger.debug("Save existing series ID: %s" % str(series.id)) - series = tseries - else: - print "Series doesn't exist (if you are not, you should be running SaveAs)" - - if is_new_series: - - - series = series_module.copy_series(series) - - - if var: - series.variable_id = var_id - series.variable_code = var.code - series.variable_name = var.name - series.speciation = var.speciation - series.variable_units_id = var.variable_unit_id - series.variable_units_name = var.variable_unit.name - series.sample_medium = var.sample_medium - series.value_type = var.value_type - series.time_support = var.time_support - series.time_units_id = var.time_unit_id - series.time_units_name = var.time_unit.name - series.data_type = var.data_type - series.general_category = var.general_category - - if method: - series.method_id = method_id - series.method_description = method.description - - if qcl: - series.quality_control_level_id = qcl_id - series.quality_control_level_code = qcl.code - ''' - dvs["LocalDateTime"] = pd.to_datetime(dvs["LocalDateTime"]) - dvs["DateTimeUTC"] = pd.to_datetime(dvs["DateTimeUTC"]) - ''' - - - - - - form = "%Y-%m-%d %H:%M:%S" - series.begin_date_time = datetime.datetime.strptime(str(np.min(dvs["LocalDateTime"])), form)#np.min(dvs["LocalDateTime"])#dvs[0].local_date_time - series.end_date_time = datetime.datetime.strptime(str(np.max(dvs["LocalDateTime"])), form)#np.max(dvs["LocalDateTime"])#dvs[-1].local_date_time - series.begin_date_time_utc = datetime.datetime.strptime(str(np.min(dvs["DateTimeUTC"])), form) #dvs[0].date_time_utc - series.end_date_time_utc = datetime.datetime.strptime(str(np.max(dvs["DateTimeUTC"])), form) #dvs[-1].date_time_utc - series.value_count = len(dvs) - - ## Override previous save - if not is_new_series: - # delete old dvs - #pass - self.memDB.series_service.delete_values_by_series(series) - - - #logger.debug("series.data_values: %s" % ([x for x in series.data_values])) - dvs.drop('ValueID', axis=1, inplace=True) - return series, dvs - - def save(self): - """ Save to an existing catalog - :param var: - :param method: - :param qcl: - :return: - """ - - series, dvs = self.updateSeries(is_new_series=False) - if self.memDB.series_service.save_series(series, dvs): - logger.debug("series saved!") - return True - else: - logger.debug("The Save was unsuccessful") - return False - - def save_as(self, var=None, method=None, qcl=None): - """ - :param var: - :param method: - :param qcl: - :return: - """ - series, dvs = self.updateSeries(var, method, qcl, is_new_series=True) - - if self.memDB.series_service.save_new_series(series, dvs): - logger.debug("series saved!") - return True - else: - logger.debug("The Save As Function was Unsuccessful") - return False - - def save_existing(self, var=None, method=None, qcl=None): - """ - :param var: - :param method: - :param qcl: - :return: - """ - series, dvs = self.updateSeries(var, method, qcl, is_new_series=False) - if self.memDB.series_service.save_series(series, dvs): - logger.debug("series saved!") - return True - else: - logger.debug("The Save As Existing Function was Unsuccessful") - return False - - def create_qcl(self, code, definition, explanation): - return self.memDB.series_service.create_qcl(code, definition, explanation) - - def create_method(self, description, link): - return self.memDB.series_service.create_method(description, link) - - def create_qualifier(self, code, definition): - return self.memDB.series_service.create_qualifier(code, definition) - - def create_variable(self, code, name, speciation, variable_unit_id, sample_medium, - value_type, is_regular, time_support, time_unit_id, data_type, general_category, no_data_value): - - return self.memDB.series_service.create_variable(code, name, speciation, variable_unit_id, sample_medium, - value_type, is_regular, time_support, time_unit_id, data_type, - general_category, no_data_value) - - def reconcile_dates(self, parent_series_id): - # FUTURE FEATURE: pull in new field data from another series and add to this series - # (i.e one series contains new field data of an edited series at a higher qcl) - pass - - diff --git a/odmtools/odmservices/export_service.py b/odmtools/odmservices/export_service.py deleted file mode 100644 index 4b6d1eb..0000000 --- a/odmtools/odmservices/export_service.py +++ /dev/null @@ -1,401 +0,0 @@ -import csv -import xml.etree.cElementTree as ET -import datetime - - -class ExportService(): - ''' - Create with the Service Manager!!! - ''' - - def __init__(self, series_service): - self._series_service = series_service - self.dt_format_str = "%m/%d/%Y %I:%M:%S %p" - - def export_series_data(self, series_id, filename, utc=False, site=False, var=False, offset=False, qual=False, - src=False, qcl=False): - series = self._series_service.get_series_by_id(series_id) - if series is None: - return False - - writer = csv.writer(open(filename, 'wb')) - print "filename: ", filename - self.write_data_header(writer, utc, site, var, offset, qual, src, qcl) - for dv in series.data_values: - self.write_data_row(writer, series, dv, utc, site, var, offset, qual, src, qcl) - - def write_data_row(self, writer, series, dv, utc, site, var, offset, qual, src, qcl): - data = [] - data.append(series.id) - data.append(dv.id) - data.append(dv.data_value) - data.append(dv.value_accuracy) - data.append(dv.local_date_time) - if utc: - data.append(dv.utc_offset) - data.append(dv.date_time_utc) - data.append(series.site_code) - if site: - data.append(series.site_name) - data.append(series.site.type) - data.append(series.site.latitude) - data.append(series.site.longitude) - data.append(series.site.spatial_ref.srs_name) - data.append(series.variable_code) - if var: - data.append(series.variable_name) - data.append(series.speciation) - data.append(series.variable_units_name) - data.append(series.variable.variable_unit.abbreviation) - data.append(series.sample_medium) - data.append(dv.offset_value) - data.append(dv.offset_type_id) - if offset: - if dv.offset_type is not None: - data.append(dv.offset_type.description) - data.append(dv.offset_type.unit.name) - else: - data.append('') - data.append('') - data.append(dv.censor_code) - data.append(dv.qualifier_id) - if qual: - if dv.qualifier is not None: - data.append(dv.qualifier.code) - data.append(dv.qualifier.description) - else: - data.append('') - data.append('') - if src: - data.append(series.organization) - data.append(series.source_description) - data.append(series.citation) - if qcl: - data.append(series.quality_control_level_code) - data.append(series.quality_control_level.definition) - data.append(series.quality_control_level.explanation) - data.append(dv.sample_id) - - writer.writerow(data) - - def write_data_header(self, writer, utc, site, var, offset, qual, src, qcl): - # Build header list - header = [] - header.append("SeriesId") - header.append("ValueId") - header.append("DataValue") - header.append("ValueAccuracy") - header.append("LocalDateTime") - if utc: - header.append("UTCOffset") - header.append("DateTimeUTC") - header.append("SiteCode") - if site: - header.append("SiteName") - header.append("SiteType") - header.append("Latitude") - header.append("Longitude") - header.append("SRSName") - header.append("VariableCode") - if var: - header.append("VariableName") - header.append("Speciation") - header.append("VariableUnitsName") - header.append("VariableUnitsAbbreviation") - header.append("SampleMedium") - header.append("OffsetValue") - header.append("OffsetTypeID") - if offset: - header.append("OffsetDescription") - header.append("OffsetUnitsName") - header.append("CensorCode") - header.append("QualifierID") - if qual: - header.append("QualifierCode") - header.append("QualifierDescription") - if src: - header.append("Organization") - header.append("SourceDescription") - header.append("Citation") - if qcl: - header.append("QualityControlLevelCode") - header.append("Definition") - header.append("Explanation") - header.append("SampleID") - - writer.writerow(header) - - def export_series_metadata(self, series_ids, filename): - if series_ids is None: - return - - root = ET.Element("Metadata") - list_root = ET.SubElement(root, "DataSeriesList") - list_root.set("Total", str(series_ids)) - - try: - with open(filename): - file_exists = True - except IOError: - file_exists = False - - if file_exists: - # Read the file into the XML tree - pass - - if isinstance(series_ids, int): - series = self._series_service.get_series_by_id(series_ids) - self.append_series_node(series, list_root) - else: - for series_id in series_ids: - series = self._series_service.get_series_by_id(series_id) - self.append_series_node(series, list_root) - - tree = ET.ElementTree(root) - tree.write(filename) - - def append_series_node(self, series, parent): - series_node = ET.SubElement(parent, "DataSeries") - series_node.set("ID", str(series.id)) - self.append_general_info(series, series_node) - self.append_site_info(series, series_node) - self.append_var_info(series, series_node) - self.append_method_source_info(series, series_node) - self.append_misc_info(series, series_node) - - return series_node - - def append_general_info(self, series, parent): - meta = series.source.iso_metadata - general_node = ET.SubElement(parent, "GeneralInformation") - topic = ET.SubElement(general_node, "TopicCategory") - topic.text = meta.topic_category - title = ET.SubElement(general_node, "Title") - title.text = meta.title - abstract = ET.SubElement(general_node, "Abstract") - abstract.text = meta.abstract - prof_version = ET.SubElement(general_node, "ProfileVersion") - prof_version.text = meta.profile_version - metadata_link = ET.SubElement(general_node, "MetadataLink") - metadata_link.text = meta.metadata_link - date = ET.SubElement(general_node, "MetadataCreationDate") - # 7/1/2013 12:17:16 PM - date.text = datetime.datetime.now().strftime(self.dt_format_str) - - def append_site_info(self, series, parent): - site = series.site - site_node = ET.SubElement(parent, "SiteInformation") - site_code = ET.SubElement(site_node, "SiteCode") - site_code.text = site.code - site_name = ET.SubElement(site_node, "SiteName") - site_name.text = site.name - site_type = ET.SubElement(site_node, "SiteType") - site_type.text = site.type - - geo_coords = ET.SubElement(site_node, "GeographicCoordinates") - latitude = ET.SubElement(geo_coords, "Latitude") - latitude.text = str(site.latitude) - longitude = ET.SubElement(geo_coords, "Longitude") - longitude.text = str(site.longitude) - srs_id = ET.SubElement(geo_coords, "SRSID") - srs_id.text = str(site.spatial_ref.srs_id) - srs_name = ET.SubElement(geo_coords, "SRSName") - srs_name.text = site.spatial_ref.srs_name - is_geo = ET.SubElement(geo_coords, "IsGeographic") - is_geo.text = str(site.spatial_ref.is_geographic) - notes = ET.SubElement(geo_coords, "Notes") - notes.text = site.spatial_ref.notes - - local_coords = ET.SubElement(site_node, "LocalCoordinates") - local_x = ET.SubElement(local_coords, "LocalX") - local_x.text = str(site.local_x) - local_y = ET.SubElement(local_coords, "LocalY") - local_y.text = str(site.local_y) - local_srs_id = ET.SubElement(local_coords, "SRSID") - local_srs_id.text = str(site.local_spatial_ref.srs_id) - local_srs_name = ET.SubElement(local_coords, "SRSName") - local_srs_name.text = site.local_spatial_ref.srs_name - local_is_geo = ET.SubElement(local_coords, "IsGeographic") - local_is_geo.text = str(site.local_spatial_ref.is_geographic) - local_notes = ET.SubElement(local_coords, "Notes") - local_notes.text = site.local_spatial_ref.notes - elevation = ET.SubElement(local_coords, "Elevation_m") - if site.elevation_m: elevation.text = str(site.elevation_m) - vert_datum = ET.SubElement(local_coords, "VerticalDatum") - if site.vertical_datum_id: vert_datum.text = str(site.vertical_datum_id) - - pos_accuracy = ET.SubElement(site_node, "PosAccuracy_m") - pos_accuracy.text = str(site.pos_accuracy_m) - state = ET.SubElement(site_node, "State") - state.text = site.state - county = ET.SubElement(site_node, "County") - county.text = site.county - comments = ET.SubElement(site_node, "Comments") - comments.text = site.comments - - def append_var_info(self, series, parent): - variable = series.variable - var_node = ET.SubElement(parent, "VariableInformation") - - var_code = ET.SubElement(var_node, "VariableCode") - var_code.text = variable.code - var_name = ET.SubElement(var_node, "VariableName") - var_name.text = variable.name - speciation = ET.SubElement(var_node, "Speciation") - speciation.text = variable.speciation - - var_units = ET.SubElement(var_node, "VariableUnits") - units_name = ET.SubElement(var_units, "UnitsName") - units_name.text = variable.variable_unit.name - units_type = ET.SubElement(var_units, "UnitsType") - units_type.text = variable.variable_unit.type - units_abbrev = ET.SubElement(var_units, "UnitsAbbreviation") - units_abbrev.text = variable.variable_unit.abbreviation - - sample_medium = ET.SubElement(var_node, "SampleMedium") - sample_medium.text = variable.sample_medium - val_type = ET.SubElement(var_node, "ValueType") - val_type.text = variable.value_type - is_reg = ET.SubElement(var_node, "IsRegular") - is_reg.text = str(variable.is_regular) - time_support = ET.SubElement(var_node, "TimeSupport") - time_support.text = str(variable.time_support) - - time_support_units = ET.SubElement(var_node, "TimeSupportUnits") - ts_units_name = ET.SubElement(time_support_units, "UnitsName") - ts_units_name.text = variable.time_unit.name - ts_units_type = ET.SubElement(time_support_units, "UnitsType") - ts_units_type.text = variable.time_unit.type - ts_units_abbrev = ET.SubElement(time_support_units, "UnitsAbbreviation") - ts_units_abbrev.text = variable.time_unit.abbreviation - - data_type = ET.SubElement(var_node, "DataType") - data_type.text = variable.data_type - gen_cat = ET.SubElement(var_node, "GeneralCategory") - gen_cat.text = variable.general_category - no_dv = ET.SubElement(var_node, "NoDataValue") - no_dv.text = str(variable.no_data_value) - - period = ET.SubElement(var_node, "PeriodOfRecord") - begin_dt = ET.SubElement(period, "BeginDateTime") - begin_dt.text = series.begin_date_time.strftime(self.dt_format_str) - end_dt = ET.SubElement(period, "EndDateTime") - end_dt.text = series.end_date_time.strftime(self.dt_format_str) - begin_dt_utc = ET.SubElement(period, "BeginDateTimeUTC") - begin_dt_utc.text = series.begin_date_time_utc.strftime(self.dt_format_str) - end_dt_utc = ET.SubElement(period, "EndDateTimeUTC") - end_dt_utc.text = series.end_date_time_utc.strftime(self.dt_format_str) - value_count = ET.SubElement(period, "ValueCount") - value_count.text = str(series.value_count) - - def append_method_source_info(self, series, parent): - method = series.method - method_node = ET.SubElement(parent, "MethodInformation") - method_desc = ET.SubElement(method_node, "MethodDescription") - method_desc.text = method.description - method_link = ET.SubElement(method_node, "MethodLink") - method_link.text = method.link - - source = series.source - source_node = ET.SubElement(parent, "SourceInformation") - org = ET.SubElement(source_node, "Organization") - org.text = source.organization - source_desc = ET.SubElement(source_node, "SourceDescription") - source_desc.text = source.description - source_link = ET.SubElement(source_node, "SourceLink") - source_link.text = source.link - - contact = ET.SubElement(source_node, "Contact") - contact_name = ET.SubElement(contact, "ContactName") - contact_name.text = source.contact_name - phone = ET.SubElement(contact, "Phone") - phone.text = source.phone - email = ET.SubElement(contact, "Email") - email.text = source.email - address = ET.SubElement(contact, "Address") - address.text = source.address - city = ET.SubElement(contact, "City") - city.text = source.city - state = ET.SubElement(contact, "State") - state.text = source.state - zip_code = ET.SubElement(contact, "ZipCode") - zip_code.text = source.zip_code - - citation = ET.SubElement(source_node, "Citation") - citation.text = source.citation - - def append_misc_info(self, series, parent): - qcl = series.quality_control_level - - qcl_node = ET.SubElement(parent, "QualityControlLevelInformation") - qcl_code = ET.SubElement(qcl_node, "QualityControlLevelCode") - qcl_code.text = qcl.code - qcl_def = ET.SubElement(qcl_node, "Definition") - qcl_def.text = qcl.definition - qcl_expl = ET.SubElement(qcl_node, "Explanation") - qcl_expl.text = qcl.explanation - - offsets_node = ET.SubElement(parent, "OffsetInformation") - offsets = self._series_service.get_offset_types_by_series_id(series.id) - for offset in offsets: - offset_id = ET.SubElement(offsets_node, "Offset") - if offset: - offset_id.set("ID", str(offset.id)) - else: - offset_id.set("ID", "") - offset_desc = ET.SubElement(offsets_node, "OffsetDescription") - if offset: offset_desc.text = offset.description - offset_units = ET.SubElement(offsets_node, "OffsetUnits") - units_name = ET.SubElement(offset_units, "UnitsName") - if offset: units_name.text = offset.unit.name - units_type = ET.SubElement(offset_units, "UnitsType") - if offset: units_type.text = offset.unit.type - units_abbrev = ET.SubElement(offset_units, "UnitsAbbreviation") - if offset: units_abbrev.text = offset.unit.abbreviation - - qualifiers_node = ET.SubElement(parent, "QualifierInformation") - qualifiers = self._series_service.get_qualifiers_by_series_id(series.id) - for qual in qualifiers: - qual_id = ET.SubElement(qualifiers_node, "Qualifier") - if qual: - qual_id.set("ID", str(qual.id)) - else: - qual_id.set("ID", "") - qual_code = ET.SubElement(qual_id, "QualiferCode") - if qual: qual_code.text = qual.code - qual_desc = ET.SubElement(qual_id, "QualifierDescription") - if qual: qual_desc.text = qual.description - - samples_node = ET.SubElement(parent, "SampleInformation") - samples = self._series_service.get_samples_by_series_id(series.id) - for sample in samples: - sample_id = ET.SubElement(samples_node, "Sample") - if sample: - sample_id.set("ID", str(sample.id)) - else: - sample_id.set("ID", "") - sample_type = ET.SubElement(sample_id, "SampleType") - if sample: sample_type.text = sample.type - lab_code = ET.SubElement(sample_id, "LabSampleCode") - if sample: lab_code.text = sample.lab_sample_code - lab_method_id = ET.SubElement(sample_id, "LabMethodID") - if sample: lab_method_id = sample.lab_method_id - - lab_method_node = ET.SubElement(parent, "LabMethodInformation") - for sample in samples: - if sample: lab_method = sample.lab_method - lab_method_id = ET.SubElement(lab_method_node, "LabMethod") - if lab_method: - lab_method_id.set("ID", str(lab_method.id)) - else: - lab_method_id.set("ID", "") - lab_name = ET.SubElement(lab_method_id, "LabName") - if lab_method: lab_name.text = lab_method.name - lab_org = ET.SubElement(lab_method_id, "LabOrganization") - if lab_method: lab_org.text = lab_method.organization - method_name = ET.SubElement(lab_method_id, "LabMethodName") - if lab_method: method_name.text = lab_method.method_name - method_desc = ET.SubElement(lab_method_id, "LabMethodDescription") - if lab_method: method_desc.text = lab_method.method_description - method_link = ET.SubElement(lab_method_id, "LabMethodLink") - if lab_method: method_link.text = lab_method.link \ No newline at end of file diff --git a/odmtools/odmservices/series_service.py b/odmtools/odmservices/series_service.py deleted file mode 100644 index 6827984..0000000 --- a/odmtools/odmservices/series_service.py +++ /dev/null @@ -1,782 +0,0 @@ -import logging - - -from sqlalchemy import distinct, func - - -from odmtools.odmdata import SessionFactory -from odmtools.odmdata import Site -from odmtools.odmdata import Variable -from odmtools.odmdata import Unit -from odmtools.odmdata import Series -from odmtools.odmdata import DataValue -from odmtools.odmdata import Qualifier -from odmtools.odmdata import OffsetType -from odmtools.odmdata import Sample -from odmtools.odmdata import Method -from odmtools.odmdata import QualityControlLevel -from odmtools.odmdata import ODMVersion -from odmtools.common.logger import LoggerTool -import pandas as pd - -tool = LoggerTool() -logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG) - - -class SeriesService(): - # Accepts a string for creating a SessionFactory, default uses odmdata/connection.cfg - def __init__(self, connection_string="", debug=False): - self._session_factory = SessionFactory(connection_string, debug) - self._edit_session = self._session_factory.get_session() - self._debug = debug - - def reset_session(self): - self._edit_session = self._session_factory.get_session() # Reset the session in order to prevent memory leaks - - def get_db_version(self): - return self._edit_session.query(ODMVersion).first().version_number - -##################### -# -# Get functions -# -##################### - - # Site methods - def get_all_sites(self): - """ - - :return: List[Sites] - """ - return self._edit_session.query(Site).order_by(Site.code).all() - - - def get_used_sites(self): - """ - Return a list of all sites that are being referenced in the Series Catalog Table - :return: List[Sites] - """ - try: - site_ids = [x[0] for x in self._edit_session.query(distinct(Series.site_id)).all()] - except: - site_ids = None - - if not site_ids: - return None - - Sites = [] - for site_id in site_ids: - Sites.append(self._edit_session.query(Site).filter_by(id=site_id).first()) - - return Sites - - - def get_site_by_id(self, site_id): - """ - return a Site object that has an id=site_id - :param site_id: integer- the identification number of the site - :return: Sites - """ - try: - return self._edit_session.query(Site).filter_by(id=site_id).first() - except: - return None - - # Variables methods - def get_used_variables(self): - """ - #get list of used variable ids - :return: List[Variables] - """ - - try: - var_ids = [x[0] for x in self._edit_session.query(distinct(Series.variable_id)).all()] - except: - var_ids = None - - if not var_ids: - return None - - Variables = [] - - #create list of variables from the list of ids - for var_id in var_ids: - Variables.append(self._edit_session.query(Variable).filter_by(id=var_id).first()) - - return Variables - - def get_all_variables(self): - """ - - :return: List[Variables] - """ - return self._edit_session.query(Variable).all() - - def get_variable_by_id(self, variable_id): - """ - - :param variable_id: int - :return: Variables - """ - try: - return self._edit_session.query(Variable).filter_by(id=variable_id).first() - except: - return None - - def get_variable_by_code(self, variable_code): - """ - - :param variable_code: str - :return: Variables - """ - try: - return self._edit_session.query(Variable).filter_by(code=variable_code).first() - except: - return None - - def get_variables_by_site_code(self, site_code): # covers NoDV, VarUnits, TimeUnits - """ - Finds all of variables at a site - :param site_code: str - :return: List[Variables] - """ - try: - var_ids = [x[0] for x in self._edit_session.query(distinct(Series.variable_id)).filter_by( - site_code=site_code).all()] - except: - var_ids = None - - variables = [] - for var_id in var_ids: - variables.append(self._edit_session.query(Variable).filter_by(id=var_id).first()) - - return variables - - # Unit methods - def get_all_units(self): - """ - - :return: List[Units] - """ - return self._edit_session.query(Unit).all() - - def get_unit_by_name(self, unit_name): - """ - - :param unit_name: str - :return: Units - """ - try: - return self._edit_session.query(Unit).filter_by(name=unit_name).first() - except: - return None - - def get_unit_by_id(self, unit_id): - """ - - :param unit_id: int - :return: Units - """ - try: - return self._edit_session.query(Unit).filter_by(id=unit_id).first() - except: - return None - - - def get_all_qualifiers(self): - """ - - :return: List[Qualifiers] - """ - result = self._edit_session.query(Qualifier).order_by(Qualifier.code).all() - return result - - def get_qualifiers_by_series_id(self, series_id): - """ - - :param series_id: - :return: - """ - subquery = self._edit_session.query(DataValue.qualifier_id).outerjoin( - Series.data_values).filter(Series.id == series_id, DataValue.qualifier_id != None).distinct().subquery() - return self._edit_session.query(Qualifier).join(subquery).distinct().all() - - #QCL methods - def get_all_qcls(self): - return self._edit_session.query(QualityControlLevel).all() - - def get_qcl_by_id(self, qcl_id): - try: - return self._edit_session.query(QualityControlLevel).filter_by(id=qcl_id).first() - except: - return None - - def get_qcl_by_code(self, qcl_code): - try: - return self._edit_session.query(QualityControlLevel).filter_by(code=qcl_code).first() - except: - return None - - # Method methods - def get_all_methods(self): - return self._edit_session.query(Method).all() - - def get_method_by_id(self, method_id): - try: - result = self._edit_session.query(Method).filter_by(id=method_id).first() - except: - result = None - return result - - def get_method_by_description(self, method_code): - try: - result = self._edit_session.query(Method).filter_by(description=method_code).first() - except: - result = None - return result - - def get_offset_types_by_series_id(self, series_id): - """ - - :param series_id: - :return: - """ - subquery = self._edit_session.query(DataValue.offset_type_id).outerjoin( - Series.data_values).filter(Series.id == series_id, DataValue.offset_type_id != None).distinct().subquery() - return self._edit_session.query(OffsetType).join(subquery).distinct().all() - - def get_samples_by_series_id(self, series_id): - """ - - :param series_id: - :return: - """ - subquery = self._edit_session.query(DataValue.sample_id).outerjoin( - Series.data_values).filter(Series.id == series_id, DataValue.sample_id != None).distinct().subquery() - return self._edit_session.query(Sample).join(subquery).distinct().all() - - # Series Catalog methods - def get_all_series(self): - """ - Returns all series as a modelObject - :return: List[Series] - """ - - #logger.debug("%s" % self._edit_session.query(Series).order_by(Series.id).all()) - return self._edit_session.query(Series).order_by(Series.id).all() - - def get_series_by_site(self , site_id): - """ - - :param site_id: int - :return: List[Series] - """ - try: - selectedSeries = self._edit_session.query(Series).filter_by(site_id=site_id).order_by(Series.id).all() - return selectedSeries - except: - return None - - def get_series_by_id(self, series_id): - """ - - :param series_id: int - :return: Series - """ - try: - return self._edit_session.query(Series).filter_by(id=series_id).first() - except Exception as e: - print e - return None - - def get_series_by_id_quint(self, site_id, var_id, method_id, source_id, qcl_id): - """ - - :param site_id: - :param var_id: - :param method_id: - :param source_id: - :param qcl_id: - :return: Series - """ - try: - return self._edit_session.query(Series).filter_by( - site_id=site_id, variable_id=var_id, method_id=method_id, - source_id=source_id, quality_control_level_id=qcl_id).first() - except: - return None - - def get_series_from_filter(self): - # Pass in probably a Series object, match it against the database - pass - - - #Data Value Methods - def get_values_by_series(self, series_id): - ''' - - :param series_id: Series id - :return: pandas dataframe - ''' - series= self.get_series_by_id(series_id) - if series: - q = self._edit_session.query(DataValue).filter_by( - site_id=series.site_id, - variable_id=series.variable_id, - method_id=series.method_id, - source_id=series.source_id, - quality_control_level_id=series.quality_control_level_id) - - query=q.statement.compile(dialect=self._session_factory.engine.dialect) - data= pd.read_sql_query(sql= query, - con = self._session_factory.engine, - params = query.params ) - #return data.set_index(data['LocalDateTime']) - return data - else: - return None - - def get_all_values_df(self): - """ - - :return: Pandas DataFrame object - """ - q = self._edit_session.query(DataValue).order_by(DataValue.local_date_time) - query = q.statement.compile(dialect=self._session_factory.engine.dialect) - data = pd.read_sql_query(sql=query, con=self._session_factory.engine, - params=query.params) - columns = list(data) - - columns.insert(0, columns.pop(columns.index("DataValue"))) - columns.insert(1, columns.pop(columns.index("LocalDateTime"))) - columns.insert(2, columns.pop(columns.index("QualifierID"))) - - data = data.ix[:, columns] - return data.set_index(data['LocalDateTime']) - - def get_all_values_list(self): - """ - - :return: - """ - result = self._edit_session.query(DataValue).order_by(DataValue.local_date_time).all() - return [x.list_repr() for x in result] - - def get_all_values(self): - return self._edit_session.query(DataValue).order_by(DataValue.local_date_time).all() - - @staticmethod - def calcSeason(row): - - month = int(row["Month"]) - - if month in [1, 2, 3]: - return 1 - elif month in[4, 5, 6]: - return 2 - elif month in [7, 8, 9]: - return 3 - elif month in [10, 11, 12]: - return 4 - - def get_all_plot_values(self): - """ - - :return: - """ - q = self._edit_session.query(DataValue.data_value.label('DataValue'), - DataValue.local_date_time.label('LocalDateTime'), - DataValue.censor_code.label('CensorCode'), - func.strftime('%m', DataValue.local_date_time).label('Month'), - func.strftime('%Y', DataValue.local_date_time).label('Year') - #DataValue.local_date_time.strftime('%m'), - #DataValue.local_date_time.strftime('%Y')) - ).order_by(DataValue.local_date_time) - query = q.statement.compile(dialect=self._session_factory.engine.dialect) - data = pd.read_sql_query(sql=query, - con=self._session_factory.engine, - params=query.params) - data["Season"] = data.apply(self.calcSeason, axis=1) - return data.set_index(data['LocalDateTime']) - - def get_plot_values(self, seriesID, noDataValue, startDate = None, endDate = None ): - """ - - :param seriesID: - :param noDataValue: - :param startDate: - :param endDate: - :return: - """ - series = self.get_series_by_id(seriesID) - - DataValues = [ - (dv.data_value, dv.local_date_time, dv.censor_code, dv.local_date_time.strftime('%m'), - dv.local_date_time.strftime('%Y')) - for dv in series.data_values - if dv.data_value != noDataValue if dv.local_date_time >= startDate if dv.local_date_time <= endDate - ] - data = pd.DataFrame(DataValues, columns=["DataValue", "LocalDateTime", "CensorCode", "Month", "Year"]) - data.set_index(data['LocalDateTime'], inplace=True) - data["Season"] = data.apply(self.calcSeason, axis=1) - return data - - - - def get_data_value_by_id(self, id): - """ - - :param id: - :return: - """ - try: - return self._edit_session.query(DataValue).filter_by(id=id).first() - except: - return None - - - - -##################### -# -#Update functions -# -##################### - def update_series(self, series): - """ - - :param series: - :return: - """ - merged_series = self._edit_session.merge(series) - self._edit_session.add(merged_series) - self._edit_session.commit() - - def update_dvs(self, dv_list): - """ - - :param dv_list: - :return: - """ - merged_dv_list = map(self._edit_session.merge, dv_list) - self._edit_session.add_all(merged_dv_list) - self._edit_session.commit() - -##################### -# -#Create functions -# -##################### - def save_series(self, series, dvs): - """ Save to an Existing Series - :param series: - :param data_values: - :return: - """ - - if self.series_exists(series): - - try: - self._edit_session.add(series) - self._edit_session.commit() - self.save_values(dvs) - except Exception as e: - self._edit_session.rollback() - raise e - logger.debug("Existing File was overwritten with new information") - return True - else: - logger.debug("There wasn't an existing file to overwrite, please select 'Save As' first") - # there wasn't an existing file to overwrite - raise Exception("Series does not exist, unable to save. Please select 'Save As'") - - - def save_new_series(self, series, dvs): - """ Create as a new catalog entry - :param series: - :param data_values: - :return: - """ - # Save As case - if self.series_exists(series): - msg = "There is already an existing file with this information. Please select 'Save' or 'Save Existing' to overwrite" - logger.debug(msg) - raise Exception(msg) - else: - try: - self._edit_session.add(series) - self._edit_session.commit() - self.save_values(dvs) - #self._edit_session.add_all(dvs) - except Exception as e: - self._edit_session.rollback() - raise e - - - logger.debug("A new series was added to the database, series id: "+str(series.id)) - return True - - def save_values(self, values): - """ - - :param values: pandas dataframe - :return: - """ - values.to_sql(name="datavalues", if_exists='append', con=self._session_factory.engine, index=False) - - - def create_new_series(self, data_values, site_id, variable_id, method_id, source_id, qcl_id): - """ - - :param data_values: - :param site_id: - :param variable_id: - :param method_id: - :param source_id: - :param qcl_id: - :return: - """ - self.update_dvs(data_values) - series = Series() - series.site_id = site_id - series.variable_id = variable_id - series.method_id = method_id - series.source_id = source_id - series.quality_control_level_id = qcl_id - - self._edit_session.add(series) - self._edit_session.commit() - return series - - def create_method(self, description, link): - """ - - :param description: - :param link: - :return: - """ - meth = Method() - meth.description = description - if link is not None: - meth.link = link - - self._edit_session.add(meth) - self._edit_session.commit() - return meth - - def create_variable_by_var(self, var): - """ - - :param var: Variable Object - :return: - """ - try: - self._edit_session.add(var) - self._edit_session.commit() - return var - except: - return None - - def create_variable( - self, code, name, speciation, variable_unit_id, sample_medium, - value_type, is_regular, time_support, time_unit_id, data_type, - general_category, no_data_value): - """ - - :param code: - :param name: - :param speciation: - :param variable_unit_id: - :param sample_medium: - :param value_type: - :param is_regular: - :param time_support: - :param time_unit_id: - :param data_type: - :param general_category: - :param no_data_value: - :return: - """ - var = Variable() - var.code = code - var.name = name - var.speciation = speciation - var.variable_unit_id = variable_unit_id - var.sample_medium = sample_medium - var.value_type = value_type - var.is_regular = is_regular - var.time_support = time_support - var.time_unit_id = time_unit_id - var.data_type = data_type - var.general_category = general_category - var.no_data_value = no_data_value - - self._edit_session.add(var) - self._edit_session.commit() - return var - - def create_qcl(self, code, definition, explanation): - """ - - :param code: - :param definition: - :param explanation: - :return: - """ - qcl = QualityControlLevel() - qcl.code = code - qcl.definition = definition - qcl.explanation = explanation - - self._edit_session.add(qcl) - self._edit_session.commit() - return qcl - - - def create_qualifier_by_qual(self, qualifier): - self._edit_session.add(qualifier) - self._edit_session.commit() - return qualifier - - def create_qualifier(self, code, description): - """ - - :param code: - :param description: - :return: - """ - qual = Qualifier() - qual.code = code - qual.description = description - - return self.create_qualifier_by_qual(qual) - -##################### -# -# Delete functions -# -##################### - - def delete_series(self, series): - """ - - :param series: - :return: - """ - self.delete_values_by_series(series) - - delete_series = self._edit_session.merge(series) - self._edit_session.delete(delete_series) - self._edit_session.commit() - - - def delete_values_by_series(self, series): - """ - - :param series: - :return: - """ - try: - return self._edit_session.query(DataValue).filter_by(site_id = series.site_id, - variable_id = series.variable_id, - method_id = series.method_id, - source_id = series.source_id, - quality_control_level_id = series.quality_control_level_id).delete() - except: - return None - - def delete_dvs(self, id_list): - """ - - :param id_list: list of ids - :return: - """ - self._edit_session.query(DataValue).filter(DataValue.id.in_(id_list)).delete(False) - -##################### -# -#Exist functions -# -##################### - - - def series_exists(self, series): - """ - - :param series: - :return: - """ - return self.series_exists_quint( - series.site_id, - series.variable_id, - series.method_id, - series.source_id, - series.quality_control_level_id - ) - - def series_exists_quint(self, site_id, var_id, method_id, source_id, qcl_id): - """ - - :param site_id: - :param var_id: - :param method_id: - :param source_id: - :param qcl_id: - :return: - """ - try: - result = self._edit_session.query(Series).filter_by( - site_id=site_id, - variable_id=var_id, - method_id=method_id, - source_id=source_id, - quality_control_level_id=qcl_id - ).one() - - return True - except: - return False - def qcl_exists(self, q): - """ - - :param q: - :return: - """ - try: - result = self._edit_session.query(QualityControlLevel).filter_by(code=q.code, definition=q.definition).one() - return True - except: - - return False - - def method_exists(self, m): - """ - - :param m: - :return: - """ - try: - result = self._edit_session.query(Method).filter_by(description=m.description).one() - return True - except: - return False - - def variable_exists(self, v): - """ - - :param v: - :return: - """ - try: - result = self._edit_session.query(Variable).filter_by(code=v.code, - name=v.name, speciation=v.speciation, - variable_unit_id=v.variable_unit_id, - sample_medium=v.sample_medium, - value_type=v.value_type, is_regular=v.is_regular, - time_support=v.time_support, - time_unit_id=v.time_unit_id, data_type=v.data_type, - general_category=v.general_category, - no_data_value=v.no_data_value).one() - return result - except: - return None \ No newline at end of file diff --git a/odmtools/odmservices/service_manager.py b/odmtools/odmservices/service_manager.py deleted file mode 100755 index f7c3f6e..0000000 --- a/odmtools/odmservices/service_manager.py +++ /dev/null @@ -1,216 +0,0 @@ -import logging -import os -import sys - -import urllib - -from sqlalchemy.exc import SQLAlchemyError#OperationalError, DBAPIError - -from odmtools.common.logger import LoggerTool -from series_service import SeriesService -from cv_service import CVService -from edit_service import EditService -from odmtools.controller import EditTools -from export_service import ExportService -from odmtools.lib.Appdirs.appdirs import user_config_dir -from odmtools.odmdata.session_factory import SessionFactory - - -tool = LoggerTool() -logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG) - - -class ServiceManager(): - def __init__(self, debug=False): - self.debug = debug - f = self._get_file('r') - self._conn_dicts = [] - self.version = 0 - self._connection_format = "%s+%s://%s:%s@%s/%s" - - # Read all lines (connections) in the connection.cfg file - while True: - line = f.readline() - if not line: - break - else: - line = line.split() - #logger.debug(line) - - if len(line) >= 5: - line_dict = {} - - line_dict['engine'] = line[0] - line_dict['user'] = line[1] - line_dict['password'] = line[2] - line_dict['address'] = line[3] - line_dict['db'] = line[4] - self._conn_dicts.append(line_dict) - - if len(self._conn_dicts) is not 0: - # The current connection defaults to the most recent (i.e. the last written to the file) - self._current_conn_dict = self._conn_dicts[-1] - else: - self._current_conn_dict = None - - f.close() - - def get_all_conn_dicts(self): - return self._conn_dicts - - def is_valid_connection(self): - if self._current_conn_dict: - conn_string = self._build_connection_string(self._current_conn_dict) - logger.debug("Conn_string: %s" % conn_string) - try: - if self.testEngine(conn_string): - return self.get_current_conn_dict() - except Exception as e: - logger.fatal("The previous database for some reason isn't accessible, please enter a new connection %s" % e.message) - return None - return None - - def get_current_conn_dict(self): - return self._current_conn_dict - - def set_current_conn_dict(self, dict): - self._current_conn_dict = dict - - def add_connection(self, conn_dict): - """conn_dict must be a dictionary with keys: engine, user, password, address, db""" - - # remove earlier connections that are identical to this one - self.delete_connection(conn_dict) - - if self.test_connection(conn_dict): - # write changes to connection file - self._conn_dicts.append(conn_dict) - self._current_conn_dict = self._conn_dicts[-1] - self._save_connections() - return True - else: - logger.error("Unable to save connection due to invalid connection to database") - return False - - - @classmethod - def testEngine(self, connection_string): - s = SessionFactory(connection_string, echo=False) - if 'mssql' in connection_string: - s.ms_test_Session().execute("Select top 1 VariableCode From Variables") - elif 'mysql' in connection_string: - s.my_test_Session().execute('Select "VariableCode" From Variables Limit 1') - elif 'postgresql' in connection_string: - #s.psql_test_Session().execute('Select "VariableCode" From "ODM2"."Variables" Limit 1') - s.psql_test_Session().execute('Select "VariableCode" From "Variables" Limit 1') - return True - - def test_connection(self, conn_dict): - try: - conn_string = self._build_connection_string(conn_dict) - if self.testEngine(conn_string) and self.get_db_version(conn_string) == '1.1.1': - return True - except SQLAlchemyError as e: - logger.error("SQLAlchemy Error: %s" % e.message) - raise e - except Exception as e: - logger.error("Error: %s" % e) - raise e - return False - - def delete_connection(self, conn_dict): - self._conn_dicts[:] = [x for x in self._conn_dicts if x != conn_dict] - - # Create and return services based on the currently active connection - def get_db_version_dict(self, conn_dict): - conn_string = self._build_connection_string(conn_dict) - self.get_db_version(conn_string) - - def get_db_version(self, conn_string): - if isinstance(conn_string, dict): - conn_string = self._build_connection_string(conn_string) - service = SeriesService(conn_string) - #if not self.version: - try: - self.version = service.get_db_version() - except Exception as e: - logger.error("Exception: %s" % e.message) - return None - return self.version - - def get_series_service(self, conn_dict=""): - conn_string = "" - if conn_dict: - conn_string = self._build_connection_string(conn_dict) - self._current_conn_dict = conn_dict - else: - conn_string = self._build_connection_string(self._current_conn_dict) - return SeriesService(conn_string, self.debug) - - def get_cv_service(self): - conn_string = self._build_connection_string(self._current_conn_dict) - return CVService(conn_string, self.debug) - - def get_edit_service(self, series_id, connection): - - return EditService(series_id, connection=connection, debug=self.debug) - - def get_record_service(self, script, series_id, connection): - return EditTools(self, script, self.get_edit_service(series_id, connection), - self._build_connection_string(self.is_valid_connection())) - - def get_export_service(self): - return ExportService(self.get_series_service()) - - ## ################### - # private variables - ## ################### - - def _get_file(self, mode): - #fn = util.resource_path('connection.config') - fn = os.path.join(user_config_dir("ODMTools", "UCHIC"), 'connection.config') - - config_file = None - try: - - if os.path.exists(fn): - config_file = open(fn, mode) - else: - os.makedirs(user_config_dir("ODMTools", "UCHIC")) - open(fn, 'w').close() - config_file = open(fn, mode) - except: - open(fn, 'w').close() - config_file = open(fn, mode) - - - return config_file - - def _build_connection_string(self, conn_dict): - driver = "" - if conn_dict['engine'] == 'mssql' and sys.platform != 'win32': - driver = "pyodbc" - quoted = urllib.quote_plus('DRIVER={FreeTDS};DSN=%s;UID=%s;PWD=%s;' % (conn_dict['address'], conn_dict['user'], conn_dict['password'])) - conn_string = 'mssql+pyodbc:///?odbc_connect={}'.format(quoted) - - else: - if conn_dict['engine'] == 'mssql': - driver = "pyodbc" - elif conn_dict['engine'] == 'mysql': - driver = "pymysql" - elif conn_dict['engine'] == 'postgresql': - driver = "psycopg2" - else: - driver = "None" - - conn_string = self._connection_format % ( - conn_dict['engine'], driver, conn_dict['user'], conn_dict['password'], conn_dict['address'], - conn_dict['db']) - return conn_string - - def _save_connections(self): - f = self._get_file('w') - for conn in self._conn_dicts: - f.write("%s %s %s %s %s\n" % (conn['engine'], conn['user'], conn['password'], conn['address'], conn['db'])) - f.close() - From c82f8f220f7bfbb2649f1f2b4a190ba781d85b26 Mon Sep 17 00:00:00 2001 From: Stephanie Reeder Date: Wed, 15 Jul 2015 16:50:01 -0600 Subject: [PATCH 02/15] change imports form sessionfactory --- odmtools/__init__.py | 2 -- odmtools/odmdata/__init__.py | 9 ++++++++- setup/__init__.py | 1 + make.py => setup/make.py | 0 4 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 setup/__init__.py rename make.py => setup/make.py (100%) diff --git a/odmtools/__init__.py b/odmtools/__init__.py index df60136..5381d99 100644 --- a/odmtools/__init__.py +++ b/odmtools/__init__.py @@ -1,6 +1,5 @@ __author__ = 'Jacob' import odmdata -import odmservices import controller import gui import lib @@ -9,7 +8,6 @@ __all__ = [ 'odmdata', - 'odmservices', 'controller', 'gui', 'lib', diff --git a/odmtools/odmdata/__init__.py b/odmtools/odmdata/__init__.py index 9e7c314..8c29f9a 100644 --- a/odmtools/odmdata/__init__.py +++ b/odmtools/odmdata/__init__.py @@ -69,4 +69,11 @@ import sys sys.path.append('/Users/stephanie/DEV/ODM2PythonAPI/') import src.api as api -from src.api.ODMconnection import SessionFactory \ No newline at end of file +from src.api.ODMconnection import SessionFactory + + + +__all__=['MemoryDatabase', + 'SessionFactory', + 'api', + ] \ No newline at end of file diff --git a/setup/__init__.py b/setup/__init__.py new file mode 100644 index 0000000..a24dbc6 --- /dev/null +++ b/setup/__init__.py @@ -0,0 +1 @@ +__author__ = 'stephanie' diff --git a/make.py b/setup/make.py similarity index 100% rename from make.py rename to setup/make.py From 3b4a413fb6eb66beed9342b6e67d8b24d3fb2a71 Mon Sep 17 00:00:00 2001 From: Stephanie Reeder Date: Fri, 14 Aug 2015 15:02:19 -0600 Subject: [PATCH 03/15] get code working with api remove extra files --- odmtools/controller/olvSeriesSelector.py | 2 +- odmtools/odmdata/__init__.py | 48 +- odmtools/odmdata/memory_database.py | 7 +- odmtools/odmdata/series.py | 129 ---- odmtools/odmservices/__init__.py | 22 +- odmtools/odmservices/edit_service.py | 583 ----------------- odmtools/odmservices/series_service.py | 782 ----------------------- odmtools/odmservices/service_manager.py | 208 ++++++ 8 files changed, 264 insertions(+), 1517 deletions(-) delete mode 100644 odmtools/odmdata/series.py delete mode 100644 odmtools/odmservices/edit_service.py delete mode 100644 odmtools/odmservices/series_service.py create mode 100755 odmtools/odmservices/service_manager.py diff --git a/odmtools/controller/olvSeriesSelector.py b/odmtools/controller/olvSeriesSelector.py index d8f0ab7..7f21f32 100644 --- a/odmtools/controller/olvSeriesSelector.py +++ b/odmtools/controller/olvSeriesSelector.py @@ -6,7 +6,7 @@ from odmtools.lib.ObjectListView import FastObjectListView, ColumnDefn from odmtools.common.logger import LoggerTool -from odmtools.odmdata import series +from odmtools.odmdata import Series tool = LoggerTool() diff --git a/odmtools/odmdata/__init__.py b/odmtools/odmdata/__init__.py index aa68242..447f960 100644 --- a/odmtools/odmdata/__init__.py +++ b/odmtools/odmdata/__init__.py @@ -66,18 +66,48 @@ ] ''' -<<<<<<< HEAD + import sys -sys.path.append('/Users/stephanie/DEV/ODM2PythonAPI/') -import src.api as api +from odmtools.odmdata import memory_database + + from src.api.ODMconnection import SessionFactory +from src.api.versionSwitcher.ODM import DataTypeCV, DataValue, GeneralCategoryCV, ISOMetadata, LabMethod, Method, \ + OffsetType,Qualifier,QualityControlLevel,Sample,SampleMediumCV,SampleTypeCV, Series, Site,SiteTypeCV,Source, \ + SpatialReference,SpeciationCV,TopicCategoryCV,Unit,ValueTypeCV,Variable,VerticalDatumCV -__all__=['MemoryDatabase', - 'SessionFactory', - 'api', - ] -======= +__all__=[ + 'SessionFactory', + 'CensorCodeCV', + 'DataTypeCV', + 'DataValue', + 'GeneralCategoryCV', + 'ISOMetadata', + 'LabMethod', + 'Method', + 'ODMVersion', + 'OffsetType', + 'Qualifier', + 'QualityControlLevel', + 'Sample', + 'SampleMediumCV', + 'SampleTypeCV', + 'Series', + 'SessionFactory', + 'Site', + 'SiteTypeCV', + 'Source', + 'SpatialReference', + 'SpeciationCV', + 'TopicCategoryCV', + 'Unit', + 'ValueTypeCV', + 'Variable', + 'VariableNameCV', + 'VerticalDatumCV', + + 'MemoryDatabase', ->>>>>>> 60dccccf7d52506b8bdb8cfd063f73ce2dad2b4b + ] diff --git a/odmtools/odmdata/memory_database.py b/odmtools/odmdata/memory_database.py index 9c3d241..00b1156 100644 --- a/odmtools/odmdata/memory_database.py +++ b/odmtools/odmdata/memory_database.py @@ -1,11 +1,10 @@ -import timeit import logging + +from sqlalchemy import bindparam + from odmtools.common.logger import LoggerTool from odmtools.odmservices import SeriesService from odmtools.odmdata import DataValue -from sqlalchemy import update, bindparam -from odmtools.common.taskServer import TaskServerMP -from multiprocessing import cpu_count, freeze_support tool = LoggerTool() logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG) diff --git a/odmtools/odmdata/series.py b/odmtools/odmdata/series.py deleted file mode 100644 index 3510dbf..0000000 --- a/odmtools/odmdata/series.py +++ /dev/null @@ -1,129 +0,0 @@ -from collections import OrderedDict # Requires Python 2.7 >= - -from sqlalchemy import Column, Integer, ForeignKey, String, Float, DateTime -from sqlalchemy.orm import relationship - -from base import Base -from site import Site -from variable import Variable -from method import Method -from source import Source -from quality_control_level import QualityControlLevel - -def copy_series(from_series): - new = Series() - new.site_id = from_series.site_id - new.site_code = from_series.site_code - new.site_name = from_series.site_name - new.variable_id = from_series.variable_id - new.variable_code = from_series.variable_code - new.variable_name = from_series.variable_name - new.speciation = from_series.speciation - new.variable_units_id = from_series.variable_units_id - new.variable_units_name = from_series.variable_units_name - new.sample_medium = from_series.sample_medium - new.value_type = from_series.value_type - new.time_support = from_series.time_support - new.time_units_id = from_series.time_units_id - new.time_units_name = from_series.time_units_name - new.data_type = from_series.data_type - new.general_category = from_series.general_category - new.method_id = from_series.method_id - new.method_description = from_series.method_description - new.source_id = from_series.source_id - new.source_description = from_series.source_description - new.organization = from_series.organization - new.citation = from_series.citation - new.quality_control_level_id = from_series.quality_control_level_id - new.quality_control_level_code = from_series.quality_control_level_code - new.begin_date_time = from_series.begin_date_time - new.begin_date_time_utc = from_series.begin_date_time_utc - new.end_date_time_utc = from_series.end_date_time_utc - new.value_count = from_series.value_count - return new -class Series(Base): - __tablename__ = 'seriescatalog' - - id = Column('SeriesID', Integer, primary_key=True) - site_id = Column('SiteID', Integer, ForeignKey('Sites.SiteID'), nullable=False) - site_code = Column('SiteCode', String) - site_name = Column('SiteName', String) - variable_id = Column('VariableID', Integer, ForeignKey('Variables.VariableID'), nullable=False) - variable_code = Column('VariableCode', String) - variable_name = Column('VariableName', String) - speciation = Column('Speciation', String) - variable_units_id = Column('VariableUnitsID', Integer) - variable_units_name = Column('VariableUnitsName', String) - sample_medium = Column('SampleMedium', String) - value_type = Column('ValueType', String) - time_support = Column('TimeSupport', Float) - time_units_id = Column('TimeUnitsID', Integer) - time_units_name = Column('TimeUnitsName', String) - data_type = Column('DataType', String) - general_category = Column('GeneralCategory', String) - method_id = Column('MethodID', Integer, ForeignKey('Methods.MethodID'), nullable=False) - method_description = Column('MethodDescription', String) - source_id = Column('SourceID', Integer, ForeignKey('Sources.SourceID'), nullable=False) - source_description = Column('SourceDescription', String) - organization = Column('Organization', String) - citation = Column('Citation', String) - quality_control_level_id = Column('QualityControlLevelID', Integer, - ForeignKey('QualityControlLevels.QualityControlLevelID'), nullable=False) - quality_control_level_code = Column('QualityControlLevelCode', String) - begin_date_time = Column('BeginDateTime', DateTime) - end_date_time = Column('EndDateTime', DateTime) - begin_date_time_utc = Column('BeginDateTimeUTC', DateTime) - end_date_time_utc = Column('EndDateTimeUTC', DateTime) - value_count = Column('ValueCount', Integer) - - data_values = relationship("DataValue", - primaryjoin="and_(DataValue.site_id == Series.site_id, " - "DataValue.variable_id == Series.variable_id, " - "DataValue.method_id == Series.method_id, " - "DataValue.source_id == Series.source_id, " - "DataValue.quality_control_level_id == Series.quality_control_level_id)", - foreign_keys="[DataValue.site_id, DataValue.variable_id, DataValue.method_id, DataValue.source_id, DataValue.quality_control_level_id]", - order_by="DataValue.local_date_time", - backref="series") - - site = relationship(Site) - variable = relationship(Variable) - method = relationship(Method) - source = relationship(Source) - quality_control_level = relationship(QualityControlLevel) - - - def __repr__(self): - return "" % (self.id, self.site_name, self.variable_code, self.variable_name) - - - def get_table_columns(self): - return self.__table__.columns.keys() - - def list_repr(self): - return [self.id, self.site_id, self.site_code, self.site_name, self.variable_id, self.variable_code, - self.variable_name, self.speciation, self.variable_units_id, self.variable_units_name, - self.sample_medium, self.value_type, self.time_support, self.time_units_id, self.time_units_name, - self.data_type, self.general_category, self.method_id, self.method_description, - self.source_id, self.source_description, self.organization, self.citation, - self.quality_control_level_id, self.quality_control_level_code, self.begin_date_time, - self.end_date_time, self.begin_date_time_utc, self.end_date_time_utc, self.value_count] - -def returnDict(): - keys = ['SeriesID', 'SiteID', 'SiteCode', 'SiteName', 'VariableID', 'VariableCode', 'VariableName', 'Speciation', - 'VariableUnitsID', 'VariableUnitsName', 'SampleMedium', 'ValueType', 'TimeSupport', 'TimeUnitsID', - 'TimeUnitsName', 'DataType', 'GeneralCategory', 'MethodID', 'MethodDescription', 'SourceID', - 'SourceDescription', 'Organization', 'Citation', 'QualityControlLevelID', 'QualityControlLevelCode', - 'BeginDateTime', 'EndDateTime', 'BeginDateTimeUTC', 'EndDateTimeUTC', 'ValueCount' - ] - values = ['id', 'site_id', 'site_code', 'site_name', 'variable_id', 'variable_code', 'variable_name', 'speciation', - 'variable_units_id', 'variable_units_name', 'sample_medium', 'value_type', 'time_support', - 'time_units_id', 'time_units_name', 'data_type', 'general_category', 'method_id', 'method_description', - 'source_id', 'source_description', 'organization', 'citation', 'quality_control_level_id', - 'quality_control_level_code', 'begin_date_time', 'end_date_time', 'begin_date_time_utc', - 'end_date_time_utc', 'value_count' - ] - return OrderedDict(zip(keys, values)) - - - diff --git a/odmtools/odmservices/__init__.py b/odmtools/odmservices/__init__.py index 6d161f5..c850d5c 100644 --- a/odmtools/odmservices/__init__.py +++ b/odmtools/odmservices/__init__.py @@ -1,13 +1,17 @@ +# from service_manager import ServiceManager +# from series_service import SeriesService +# from cv_service import CVService +# from edit_service import EditService +# from export_service import ExportService +# +# # need to explicitly import these for pyinstaller +# import pymysql +# import pyodbc +# #import psycopg2 +import sys +sys.path.append('/Users/stephanie/DEV/ODM2PythonAPI/') +from src.api.ODM1_1_1.services import SeriesService, EditService, CVService, ExportService from service_manager import ServiceManager -from series_service import SeriesService -from cv_service import CVService -from edit_service import EditService -from export_service import ExportService - -# need to explicitly import these for pyinstaller -import pymysql -import pyodbc -#import psycopg2 __all__ = [ 'EditService', diff --git a/odmtools/odmservices/edit_service.py b/odmtools/odmservices/edit_service.py deleted file mode 100644 index 4c625ab..0000000 --- a/odmtools/odmservices/edit_service.py +++ /dev/null @@ -1,583 +0,0 @@ -import sqlite3 - - -from odmtools.odmdata import DataValue -from series_service import SeriesService - -from odmtools.odmdata import series as series_module - -import pandas as pd -import datetime -import numpy as np - -import logging -from odmtools.common.logger import LoggerTool - -tool = LoggerTool() -logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG) - - -class EditService(): - # Mutual exclusion: cursor, or connection_string - def __init__(self, series_id, connection=None, connection_string="", debug=False): - ''' - - :param series_id: - :param connection: memory database, contains connection to remote database - :param connection_string: connection to remote database - :param debug: - :return: - ''' - - self._series_id = series_id - self._filter_from_selection = False - self._debug = debug - - if connection_string is "" and connection is not None: - self.memDB= connection - #self._series_service = self.memDB.series_service#SeriesService(connection_string, debug) - - elif connection_string is not "" and connection is None: - from odmtools.odmdata import MemoryDatabase - self.memDB= MemoryDatabase()#(series_service) - self.memDB.set_series_service(SeriesService(connection_string, False)) - - - else: - logger.error("must send in either a remote db connection string or a memory database object") - - logger.debug("Initializing Memory Database") - self.memDB.initEditValues(series_id) - logger.debug("Finished Initializing Memory Database") - self._populate_series() - self.reset_filter() - - def get_series_service(self): - return self.memDB.series_service - - def _populate_series(self): - # [(ID, value, datetime), ...] - #self._cursor.execute("SELECT ValueID, DataValue, LocalDateTime FROM DataValues ORDER BY LocalDateTime") - - self._series_points_df = self.memDB.getDataValuesDF() - - - def _test_filter_previous(self): - - ''' - if not self._filter_from_selection: - self.reset_filter() - ''' - - df = None - - if not self._filter_from_selection: - df = self._series_points_df - else: - df = self.filtered_dataframe - - # Ensure that we're not working with an empty dataframe - - if isinstance(df, pd.DataFrame): - if df.empty: - return self._series_points_df - else: - if not df: - return self._series_points_df - - return df - - def datetime2dataframe(self, datetime_list): - """ Converts datetime_list to a pandas Dataframe - - - :param datetime_list: - :return Pandas.DataFrame: - """ - - result = None - - if isinstance(datetime_list, list): - - result = pd.DataFrame(datetime_list, columns=["LocalDateTime"]) - - result.set_index("LocalDateTime", inplace=True) - - return result - - ################### - # Stubs - ################### - def selectPointsStub(self): - """ - :param filtered_dataframe: - :return: - """ - - ## Convert dataframe into list of datetimes - - filtered_dataframe = self.get_filtered_points() - if isinstance(filtered_dataframe, pd.DataFrame): - if not filtered_dataframe.empty: - datetime_list = filtered_dataframe.index.to_pydatetime() - return datetime_list.tolist() - return [] - - ################### - # Filters - ################### - # operator is a character, either '<' or '>' - def filter_value(self, value, ops): - df = self._test_filter_previous() - - if ops == '>': - self.filtered_dataframe = df[df['DataValue'] > value] - - if ops == '<': - self.filtered_dataframe = df[df['DataValue'] < value] - - - def filter_date(self, before, after): - df = self._test_filter_previous() - if before and after: - self.filtered_dataframe = df[(df.index < before) & (df.index > after)] - - # Data Gaps - def data_gaps(self, value, time_period): - df = self._test_filter_previous() - - time_units = { - 'second': 's', - 'minute': 'm', - 'hour': 'h', - 'day': 'D', - 'week': 'W', - 'month': 'M', - 'year': 'Y' - } - - # make a copy of the dataframe in order to modify it to be in the form we need to determine data gaps - copy_df = df - copy_df['datetime'] = df.index - copy_df['dateprev'] = copy_df['datetime'].shift() - - # ensure that 'value' is an integer - if not isinstance(value, int): - value = int(value) - - # create a bool column indicating which rows meet condition - filtered_results = copy_df['datetime'].diff() >= np.timedelta64(value, time_units[time_period]) - - # filter on rows that passed previous condition - copy_df = copy_df[filtered_results] - - # merge values and remove duplicates. this hack allows for both values to be marked when selecting data gaps - newdf = pd.concat([copy_df['datetime'], copy_df['dateprev']], join='inner') - self.filtered_dataframe = df[df.index.isin(newdf.drop_duplicates().dropna())] - - # clean up - del copy_df - del filtered_results - del newdf - - def change_value_threshold(self, value, operator): - - df = self._test_filter_previous() - - # make a copy of the dataframe in order to modify it to be in the form we need to determine data gaps - copy_df = df - copy_df['values'] = df['DataValue'] - copy_df['diff'] = copy_df['values'].shift() - copy_df["diff_date"] = copy_df['LocalDateTime'].shift() - copy_df['change_threshold'] = abs(df['values'] - df['diff']) - - if not isinstance(value, float): - logger.error("Need to have a float") - return - - copy_df['threshold'] = value - - if operator == ">": - copy_df['matches'] = df['change_threshold'] >= copy_df['threshold'] - - if operator == "<": - copy_df['matches'] = df['change_threshold'] <= copy_df['threshold'] - - filtered_df = copy_df[copy_df['matches']] - tmplist = filtered_df['diff_date'].tolist() + filtered_df.index.tolist() - del copy_df - self.filtered_dataframe = df[df.index.isin(tmplist)] - - - def select_points_tf(self, tf_list): - self._filter_list = tf_list - - #def select_points(self, id_list=[], datetime_list=[]): - def select_points(self, id_list=[], dataframe=[]): - #self.reset_filter() - - # This should be either one or the other. If it's both, id is used first. - # If neither are set this function does nothing. - - if len(id_list) > 0: - for i in range(len(self._series_points)): - if self._series_points[i][0] in id_list: - self._filter_list[i] = True - - if isinstance(dataframe, pd.DataFrame): - result = dataframe.index.astype(datetime.datetime) - self.filtered_dataframe = self._series_points_df[self._series_points_df.index.isin(dataframe.index)] - - - def reset_filter(self): - self.filtered_dataframe = None - - def filter_from_previous(self, value): - self._filter_from_selection = value - - def get_toggle(self): - return self._filter_from_selection - - - ################### - # Gets - ################### - def get_series(self): - return self.memDB.series_service.get_series_by_id(self._series_id) - - def get_series_points(self): - # all point in the series - return self._series_points - - def get_series_points_df(self): - """ - :return Pandas DataFrame: - """ - return self._series_points_df - - def get_filtered_points(self): - """ - :return Pandas DataFrame: - """ - if isinstance(self.filtered_dataframe, pd.DataFrame): - if self.filtered_dataframe.empty: - return None - else: - if not self.filtered_dataframe: - return None - if len(self.filtered_dataframe) > 0: - return self.filtered_dataframe - return None - - def get_filtered_dates(self): - return self.filtered_dataframe - - def get_filter_list(self): - # true or false list the length of the entire series. true indicate the point is selected - return self._filter_list - - def get_qcl(self, qcl_id): - return self.memDB.series_service.get_qcl_by_id(qcl_id) - - def get_method(self, method_id): - return self.memDB.series_service.get_method_by_id(method_id) - - def get_variable(self, variable_id): - logger.debug(variable_id) - return self.memDB.series_service.get_variable_by_id(variable_id) - - - ################# - # Edits - ################# - - def change_value(self, value, operator): - filtered_points = self.get_filtered_points() - - ids = filtered_points.index.tolist() - self.memDB.updateValue(ids, operator, float(value)) - self._populate_series() - - ## update filtered_dataframe - self.filtered_dataframe = self._series_points_df[self._series_points_df.index.isin(ids)] - - def add_points(self, points): - # todo: add the ability to send in multiple datetimes to a single 'point' - - self.memDB.addPoints(points) - - self._populate_series() - self.reset_filter() - - def delete_points(self): - filtered_points = self.get_filtered_points() - if not filtered_points.empty: - values = filtered_points.index.tolist() - - self.memDB.delete(values) - self._populate_series() - self.filtered_dataframe = None - - def interpolate(self): - ''' - In [75]: ser = Series(np.sort(np.random.uniform(size=100))) - # interpolate at new_index - In [76]: new_index = ser.index | Index([49.25, 49.5, 49.75, 50.25, 50.5, 50.75]) - In [77]: interp_s = ser.reindex(new_index).interpolate(method='pchip') - ''' - - tmp_filter_list =self.get_filtered_points() - df = self._series_points_df - issel = df.index.isin(tmp_filter_list.index) - - mdf = df["DataValue"].mask(issel) - mdf.interpolate(method = "time", inplace=True) - tmp_filter_list["DataValue"]=mdf[issel] - ids = tmp_filter_list.index.tolist() - - #update_list = [(row["DataValue"], row["ValueID"]) for index, row in tmp_filter_list.iterrows()] - update_list = [{"value": row["DataValue"], "id": index} for index, row in tmp_filter_list.iterrows()] - - self.memDB.update(update_list) - - - self._populate_series() - - self.filtered_dataframe = self._series_points_df[self._series_points_df.index.isin(ids)] - - def drift_correction(self, gap_width): - if self.isOneGroup(): - tmp_filter_list =self.get_filtered_points() - startdate =tmp_filter_list.index[0] - x_l = (tmp_filter_list.index[-1]-startdate).total_seconds() - - # y_n = y_0 + G(x_i / x_l) - f = lambda row : row["DataValue"]+(gap_width * ((row.name-startdate).total_seconds() / x_l)) - tmp_filter_list["DataValue"]=tmp_filter_list.apply(f, axis = 1) - - update_list = [{"value": row["DataValue"], "id":index} for index, row in tmp_filter_list.iterrows()] - - ids = tmp_filter_list.index.tolist() - self.memDB.update(update_list) - - - self._populate_series() - - self.filtered_dataframe = self._series_points_df[self._series_points_df.index.isin(ids)] - return True - return False - - def isOneGroup(self): - - issel = self._series_points_df.index.isin(self.get_filtered_points().index) - - found_group = False - count = 0 - - for x in issel: - if x: - if not found_group: - found_group=True - count =count+1 - else: - found_group = False - - if count >1: - return False - if count == 1: - return True - - - def flag(self, qualifier_id): - - filtered_points = self.get_filtered_points() - ''' - query = "UPDATE DataValues SET QualifierID = %s WHERE ValueID = ?" % (qualifier_id) - #self._cursor.executemany(query, [(str(x[0]),) for x in filtered_points]) - self._cursor.executemany(query, [(str(x),) for x in filtered_points["ValueID"].astype(int).tolist()]) - ''' - self.memDB.updateFlag(filtered_points.index.astype(int).tolist(), qualifier_id) - - ################### - # Save/Restore - ################### - - def restore(self): - self.memDB.rollback() - - self._populate_series() - self.reset_filter() - - def updateSeries(self, var=None, method=None, qcl=None, is_new_series=False): - """ - - :param var: - :param method: - :param qcl: - :param is_new_series: - :return: - """ - - var_id = var.id if var is not None else None - method_id = method.id if method is not None else None - qcl_id = qcl.id if qcl is not None else None - #self.memDB.changeSeriesIDs(var_id, method_id, qcl_id) - dvs = self.memDB.getDataValuesDF() - if var_id is not None: - dvs["VariableID"] = var_id - if method_id is not None: - dvs["MethodID"] = method_id - if qcl_id is not None: - dvs["QualityControlLevelID"] = qcl_id - - - - #if is new series remove valueids - #if is_new_series: - dvs["ValueID"] = None - ''' - for dv in dvs: - dv.id = None - ''' - - series = self.memDB.series_service.get_series_by_id(self._series_id) - logger.debug("original editing series id: %s" % str(series.id)) - - if (var or method or qcl ): - tseries = self.memDB.series_service.get_series_by_id_quint(site_id=int(series.site_id), - var_id=var_id if var else int(series.variable_id), - method_id=method_id if method else int( - series.method_id), - source_id=series.source_id, - qcl_id=qcl_id if qcl else int( - series.quality_control_level_id)) - if tseries: - logger.debug("Save existing series ID: %s" % str(series.id)) - series = tseries - else: - print "Series doesn't exist (if you are not, you should be running SaveAs)" - - if is_new_series: - - - series = series_module.copy_series(series) - - - if var: - series.variable_id = var_id - series.variable_code = var.code - series.variable_name = var.name - series.speciation = var.speciation - series.variable_units_id = var.variable_unit_id - series.variable_units_name = var.variable_unit.name - series.sample_medium = var.sample_medium - series.value_type = var.value_type - series.time_support = var.time_support - series.time_units_id = var.time_unit_id - series.time_units_name = var.time_unit.name - series.data_type = var.data_type - series.general_category = var.general_category - - if method: - series.method_id = method_id - series.method_description = method.description - - if qcl: - series.quality_control_level_id = qcl_id - series.quality_control_level_code = qcl.code - ''' - dvs["LocalDateTime"] = pd.to_datetime(dvs["LocalDateTime"]) - dvs["DateTimeUTC"] = pd.to_datetime(dvs["DateTimeUTC"]) - ''' - - - - - - form = "%Y-%m-%d %H:%M:%S" - series.begin_date_time = datetime.datetime.strptime(str(np.min(dvs["LocalDateTime"])), form)#np.min(dvs["LocalDateTime"])#dvs[0].local_date_time - series.end_date_time = datetime.datetime.strptime(str(np.max(dvs["LocalDateTime"])), form)#np.max(dvs["LocalDateTime"])#dvs[-1].local_date_time - series.begin_date_time_utc = datetime.datetime.strptime(str(np.min(dvs["DateTimeUTC"])), form) #dvs[0].date_time_utc - series.end_date_time_utc = datetime.datetime.strptime(str(np.max(dvs["DateTimeUTC"])), form) #dvs[-1].date_time_utc - series.value_count = len(dvs) - - ## Override previous save - if not is_new_series: - # delete old dvs - #pass - self.memDB.series_service.delete_values_by_series(series) - - - #logger.debug("series.data_values: %s" % ([x for x in series.data_values])) - dvs.drop('ValueID', axis=1, inplace=True) - return series, dvs - - def save(self): - """ Save to an existing catalog - :param var: - :param method: - :param qcl: - :return: - """ - - series, dvs = self.updateSeries(is_new_series=False) - if self.memDB.series_service.save_series(series, dvs): - logger.debug("series saved!") - return True - else: - logger.debug("The Save was unsuccessful") - return False - - def save_as(self, var=None, method=None, qcl=None): - """ - :param var: - :param method: - :param qcl: - :return: - """ - series, dvs = self.updateSeries(var, method, qcl, is_new_series=True) - - if self.memDB.series_service.save_new_series(series, dvs): - logger.debug("series saved!") - return True - else: - logger.debug("The Save As Function was Unsuccessful") - return False - - def save_existing(self, var=None, method=None, qcl=None): - """ - :param var: - :param method: - :param qcl: - :return: - """ - series, dvs = self.updateSeries(var, method, qcl, is_new_series=False) - if self.memDB.series_service.save_series(series, dvs): - logger.debug("series saved!") - return True - else: - logger.debug("The Save As Existing Function was Unsuccessful") - return False - - def create_qcl(self, code, definition, explanation): - return self.memDB.series_service.create_qcl(code, definition, explanation) - - def create_method(self, description, link): - return self.memDB.series_service.create_method(description, link) - - def create_qualifier(self, code, definition): - return self.memDB.series_service.create_qualifier(code, definition) - - def create_variable(self, code, name, speciation, variable_unit_id, sample_medium, - value_type, is_regular, time_support, time_unit_id, data_type, general_category, no_data_value): - - return self.memDB.series_service.create_variable(code, name, speciation, variable_unit_id, sample_medium, - value_type, is_regular, time_support, time_unit_id, data_type, - general_category, no_data_value) - - def reconcile_dates(self, parent_series_id): - # FUTURE FEATURE: pull in new field data from another series and add to this series - # (i.e one series contains new field data of an edited series at a higher qcl) - pass - - diff --git a/odmtools/odmservices/series_service.py b/odmtools/odmservices/series_service.py deleted file mode 100644 index 7c8ccc4..0000000 --- a/odmtools/odmservices/series_service.py +++ /dev/null @@ -1,782 +0,0 @@ -import logging - - -from sqlalchemy import distinct, func - - -from odmtools.odmdata import SessionFactory -from odmtools.odmdata import Site -from odmtools.odmdata import Variable -from odmtools.odmdata import Unit -from odmtools.odmdata import Series -from odmtools.odmdata import DataValue -from odmtools.odmdata import Qualifier -from odmtools.odmdata import OffsetType -from odmtools.odmdata import Sample -from odmtools.odmdata import Method -from odmtools.odmdata import QualityControlLevel -from odmtools.odmdata import ODMVersion -from odmtools.common.logger import LoggerTool -import pandas as pd - -tool = LoggerTool() -logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG) - - -class SeriesService(): - # Accepts a string for creating a SessionFactory, default uses odmdata/connection.cfg - def __init__(self, connection_string="", debug=False): - self._session_factory = SessionFactory(connection_string, debug) - self._edit_session = self._session_factory.get_session() - self._debug = debug - - def reset_session(self): - self._edit_session = self._session_factory.get_session() # Reset the session in order to prevent memory leaks - - def get_db_version(self): - return self._edit_session.query(ODMVersion).first().version_number - -##################### -# -# Get functions -# -##################### - - # Site methods - def get_all_sites(self): - """ - - :return: List[Sites] - """ - return self._edit_session.query(Site).order_by(Site.code).all() - - - def get_used_sites(self): - """ - Return a list of all sites that are being referenced in the Series Catalog Table - :return: List[Sites] - """ - try: - site_ids = [x[0] for x in self._edit_session.query(distinct(Series.site_id)).all()] - except: - site_ids = None - - if not site_ids: - return None - - Sites = [] - for site_id in site_ids: - Sites.append(self._edit_session.query(Site).filter_by(id=site_id).first()) - - return Sites - - - def get_site_by_id(self, site_id): - """ - return a Site object that has an id=site_id - :param site_id: integer- the identification number of the site - :return: Sites - """ - try: - return self._edit_session.query(Site).filter_by(id=site_id).first() - except: - return None - - # Variables methods - def get_used_variables(self): - """ - #get list of used variable ids - :return: List[Variables] - """ - - try: - var_ids = [x[0] for x in self._edit_session.query(distinct(Series.variable_id)).all()] - except: - var_ids = None - - if not var_ids: - return None - - Variables = [] - - #create list of variables from the list of ids - for var_id in var_ids: - Variables.append(self._edit_session.query(Variable).filter_by(id=var_id).first()) - - return Variables - - def get_all_variables(self): - """ - - :return: List[Variables] - """ - return self._edit_session.query(Variable).all() - - def get_variable_by_id(self, variable_id): - """ - - :param variable_id: int - :return: Variables - """ - try: - return self._edit_session.query(Variable).filter_by(id=variable_id).first() - except: - return None - - def get_variable_by_code(self, variable_code): - """ - - :param variable_code: str - :return: Variables - """ - try: - return self._edit_session.query(Variable).filter_by(code=variable_code).first() - except: - return None - - def get_variables_by_site_code(self, site_code): # covers NoDV, VarUnits, TimeUnits - """ - Finds all of variables at a site - :param site_code: str - :return: List[Variables] - """ - try: - var_ids = [x[0] for x in self._edit_session.query(distinct(Series.variable_id)).filter_by( - site_code=site_code).all()] - except: - var_ids = None - - variables = [] - for var_id in var_ids: - variables.append(self._edit_session.query(Variable).filter_by(id=var_id).first()) - - return variables - - # Unit methods - def get_all_units(self): - """ - - :return: List[Units] - """ - return self._edit_session.query(Unit).all() - - def get_unit_by_name(self, unit_name): - """ - - :param unit_name: str - :return: Units - """ - try: - return self._edit_session.query(Unit).filter_by(name=unit_name).first() - except: - return None - - def get_unit_by_id(self, unit_id): - """ - - :param unit_id: int - :return: Units - """ - try: - return self._edit_session.query(Unit).filter_by(id=unit_id).first() - except: - return None - - - def get_all_qualifiers(self): - """ - - :return: List[Qualifiers] - """ - result = self._edit_session.query(Qualifier).order_by(Qualifier.code).all() - return result - - def get_qualifiers_by_series_id(self, series_id): - """ - - :param series_id: - :return: - """ - subquery = self._edit_session.query(DataValue.qualifier_id).outerjoin( - Series.data_values).filter(Series.id == series_id, DataValue.qualifier_id != None).distinct().subquery() - return self._edit_session.query(Qualifier).join(subquery).distinct().all() - - #QCL methods - def get_all_qcls(self): - return self._edit_session.query(QualityControlLevel).all() - - def get_qcl_by_id(self, qcl_id): - try: - return self._edit_session.query(QualityControlLevel).filter_by(id=qcl_id).first() - except: - return None - - def get_qcl_by_code(self, qcl_code): - try: - return self._edit_session.query(QualityControlLevel).filter_by(code=qcl_code).first() - except: - return None - - # Method methods - def get_all_methods(self): - return self._edit_session.query(Method).all() - - def get_method_by_id(self, method_id): - try: - result = self._edit_session.query(Method).filter_by(id=method_id).first() - except: - result = None - return result - - def get_method_by_description(self, method_code): - try: - result = self._edit_session.query(Method).filter_by(description=method_code).first() - except: - result = None - return result - - def get_offset_types_by_series_id(self, series_id): - """ - - :param series_id: - :return: - """ - subquery = self._edit_session.query(DataValue.offset_type_id).outerjoin( - Series.data_values).filter(Series.id == series_id, DataValue.offset_type_id != None).distinct().subquery() - return self._edit_session.query(OffsetType).join(subquery).distinct().all() - - def get_samples_by_series_id(self, series_id): - """ - - :param series_id: - :return: - """ - subquery = self._edit_session.query(DataValue.sample_id).outerjoin( - Series.data_values).filter(Series.id == series_id, DataValue.sample_id != None).distinct().subquery() - return self._edit_session.query(Sample).join(subquery).distinct().all() - - # Series Catalog methods - def get_all_series(self): - """ - Returns all series as a modelObject - :return: List[Series] - """ - - #logger.debug("%s" % self._edit_session.query(Series).order_by(Series.id).all()) - return self._edit_session.query(Series).order_by(Series.id).all() - - def get_series_by_site(self , site_id): - """ - - :param site_id: int - :return: List[Series] - """ - try: - selectedSeries = self._edit_session.query(Series).filter_by(site_id=site_id).order_by(Series.id).all() - return selectedSeries - except: - return None - - def get_series_by_id(self, series_id): - """ - - :param series_id: int - :return: Series - """ - try: - return self._edit_session.query(Series).filter_by(id=series_id).first() - except Exception as e: - print e - return None - - def get_series_by_id_quint(self, site_id, var_id, method_id, source_id, qcl_id): - """ - - :param site_id: - :param var_id: - :param method_id: - :param source_id: - :param qcl_id: - :return: Series - """ - try: - return self._edit_session.query(Series).filter_by( - site_id=site_id, variable_id=var_id, method_id=method_id, - source_id=source_id, quality_control_level_id=qcl_id).first() - except: - return None - - def get_series_from_filter(self): - # Pass in probably a Series object, match it against the database - pass - - - #Data Value Methods - def get_values_by_series(self, series_id): - ''' - - :param series_id: Series id - :return: pandas dataframe - ''' - series= self.get_series_by_id(series_id) - if series: - q = self._edit_session.query(DataValue).filter_by( - site_id=series.site_id, - variable_id=series.variable_id, - method_id=series.method_id, - source_id=series.source_id, - quality_control_level_id=series.quality_control_level_id) - - query=q.statement.compile(dialect=self._session_factory.engine.dialect) - data= pd.read_sql_query(sql= query, - con = self._session_factory.engine, - params = query.params ) - #return data.set_index(data['LocalDateTime']) - return data - else: - return None - - def get_all_values_df(self): - """ - - :return: Pandas DataFrame object - """ - q = self._edit_session.query(DataValue).order_by(DataValue.local_date_time) - query = q.statement.compile(dialect=self._session_factory.engine.dialect) - data = pd.read_sql_query(sql=query, con=self._session_factory.engine, - params=query.params) - columns = list(data) - - columns.insert(0, columns.pop(columns.index("DataValue"))) - columns.insert(1, columns.pop(columns.index("LocalDateTime"))) - columns.insert(2, columns.pop(columns.index("QualifierID"))) - - data = data.ix[:, columns] - return data.set_index(data['LocalDateTime']) - - def get_all_values_list(self): - """ - - :return: - """ - result = self._edit_session.query(DataValue).order_by(DataValue.local_date_time).all() - return [x.list_repr() for x in result] - - def get_all_values(self): - return self._edit_session.query(DataValue).order_by(DataValue.local_date_time).all() - - @staticmethod - def calcSeason(row): - - month = int(row["Month"]) - - if month in [1, 2, 3]: - return 1 - elif month in[4, 5, 6]: - return 2 - elif month in [7, 8, 9]: - return 3 - elif month in [10, 11, 12]: - return 4 - - def get_all_plot_values(self): - """ - - :return: - """ - q = self._edit_session.query(DataValue.data_value.label('DataValue'), - DataValue.local_date_time.label('LocalDateTime'), - DataValue.censor_code.label('CensorCode'), - func.strftime('%m', DataValue.local_date_time).label('Month'), - func.strftime('%Y', DataValue.local_date_time).label('Year') - #DataValue.local_date_time.strftime('%m'), - #DataValue.local_date_time.strftime('%Y')) - ).order_by(DataValue.local_date_time) - query = q.statement.compile(dialect=self._session_factory.engine.dialect) - data = pd.read_sql_query(sql=query, - con=self._session_factory.engine, - params=query.params) - data["Season"] = data.apply(self.calcSeason, axis=1) - return data.set_index(data['LocalDateTime']) - - def get_plot_values(self, seriesID, noDataValue, startDate = None, endDate = None ): - """ - - :param seriesID: - :param noDataValue: - :param startDate: - :param endDate: - :return: - """ - series = self.get_series_by_id(seriesID) - - DataValues = [ - (dv.data_value, dv.local_date_time, dv.censor_code, dv.local_date_time.strftime('%m'), - dv.local_date_time.strftime('%Y')) - for dv in series.data_values - if dv.data_value != noDataValue if dv.local_date_time >= startDate if dv.local_date_time <= endDate - ] - data = pd.DataFrame(DataValues, columns=["DataValue", "LocalDateTime", "CensorCode", "Month", "Year"]) - data.set_index(data['LocalDateTime'], inplace=True) - data["Season"] = data.apply(self.calcSeason, axis=1) - return data - - - - def get_data_value_by_id(self, id): - """ - - :param id: - :return: - """ - try: - return self._edit_session.query(DataValue).filter_by(id=id).first() - except: - return None - - - - -##################### -# -#Update functions -# -##################### - def update_series(self, series): - """ - - :param series: - :return: - """ - merged_series = self._edit_session.merge(series) - self._edit_session.add(merged_series) - self._edit_session.commit() - - def update_dvs(self, dv_list): - """ - - :param dv_list: - :return: - """ - merged_dv_list = map(self._edit_session.merge, dv_list) - self._edit_session.add_all(merged_dv_list) - self._edit_session.commit() - -##################### -# -#Create functions -# -##################### - def save_series(self, series, dvs): - """ Save to an Existing Series - :param series: - :param data_values: - :return: - """ - - if self.series_exists(series): - - try: - self._edit_session.add(series) - self._edit_session.commit() - self.save_values(dvs) - except Exception as e: - self._edit_session.rollback() - raise e - logger.debug("Existing File was overwritten with new information") - return True - else: - logger.debug("There wasn't an existing file to overwrite, please select 'Save As' first") - # there wasn't an existing file to overwrite - raise Exception("Series does not exist, unable to save. Please select 'Save As'") - - - def save_new_series(self, series, dvs): - """ Create as a new catalog entry - :param series: - :param data_values: - :return: - """ - # Save As case - if self.series_exists(series): - msg = "There is already an existing file with this information. Please select 'Save' or 'Save Existing' to overwrite" - logger.debug(msg) - raise Exception(msg) - else: - try: - self._edit_session.add(series) - self._edit_session.commit() - self.save_values(dvs) - #self._edit_session.add_all(dvs) - except Exception as e: - self._edit_session.rollback() - raise e - - - logger.debug("A new series was added to the database, series id: "+str(series.id)) - return True - - def save_values(self, values): - """ - - :param values: pandas dataframe - :return: - """ - values.to_sql(name="datavalues", if_exists='append', con=self._session_factory.engine, index=False) - - - def create_new_series(self, data_values, site_id, variable_id, method_id, source_id, qcl_id): - """ - - :param data_values: - :param site_id: - :param variable_id: - :param method_id: - :param source_id: - :param qcl_id: - :return: - """ - self.update_dvs(data_values) - series = Series() - series.site_id = site_id - series.variable_id = variable_id - series.method_id = method_id - series.source_id = source_id - series.quality_control_level_id = qcl_id - - self._edit_session.add(series) - self._edit_session.commit() - return series - - def create_method(self, description, link): - """ - - :param description: - :param link: - :return: - """ - meth = Method() - meth.description = description - if link is not None: - meth.link = link - - self._edit_session.add(meth) - self._edit_session.commit() - return meth - - def create_variable_by_var(self, var): - """ - - :param var: Variable Object - :return: - """ - try: - self._edit_session.add(var) - self._edit_session.commit() - return var - except: - return None - - def create_variable( - self, code, name, speciation, variable_unit_id, sample_medium, - value_type, is_regular, time_support, time_unit_id, data_type, - general_category, no_data_value): - """ - - :param code: - :param name: - :param speciation: - :param variable_unit_id: - :param sample_medium: - :param value_type: - :param is_regular: - :param time_support: - :param time_unit_id: - :param data_type: - :param general_category: - :param no_data_value: - :return: - """ - var = Variable() - var.code = code - var.name = name - var.speciation = speciation - var.variable_unit_id = variable_unit_id - var.sample_medium = sample_medium - var.value_type = value_type - var.is_regular = is_regular - var.time_support = time_support - var.time_unit_id = time_unit_id - var.data_type = data_type - var.general_category = general_category - var.no_data_value = no_data_value - - self._edit_session.add(var) - self._edit_session.commit() - return var - - def create_qcl(self, code, definition, explanation): - """ - - :param code: - :param definition: - :param explanation: - :return: - """ - qcl = QualityControlLevel() - qcl.code = code - qcl.definition = definition - qcl.explanation = explanation - - self._edit_session.add(qcl) - self._edit_session.commit() - return qcl - - - def create_qualifier_by_qual(self, qualifier): - self._edit_session.add(qualifier) - self._edit_session.commit() - return qualifier - - def create_qualifier(self, code, description): - """ - - :param code: - :param description: - :return: - """ - qual = Qualifier() - qual.code = code - qual.description = description - - return self.create_qualifier_by_qual(qual) - -##################### -# -# Delete functions -# -##################### - - def delete_series(self, series): - """ - - :param series: - :return: - """ - self.delete_values_by_series(series) - - delete_series = self._edit_session.merge(series) - self._edit_session.delete(delete_series) - self._edit_session.commit() - - - def delete_values_by_series(self, series): - """ - - :param series: - :return: - """ - try: - return self._edit_session.query(DataValue).filter_by(site_id = series.site_id, - variable_id = series.variable_id, - method_id = series.method_id, - source_id = series.source_id, - quality_control_level_id = series.quality_control_level_id).delete() - except: - return None - - def delete_dvs(self, id_list): - """ - - :param id_list: list of ids - :return: - """ - self._edit_session.query(DataValue).filter(DataValue.local_date_time.in_(id_list)).delete(False) - -##################### -# -#Exist functions -# -##################### - - - def series_exists(self, series): - """ - - :param series: - :return: - """ - return self.series_exists_quint( - series.site_id, - series.variable_id, - series.method_id, - series.source_id, - series.quality_control_level_id - ) - - def series_exists_quint(self, site_id, var_id, method_id, source_id, qcl_id): - """ - - :param site_id: - :param var_id: - :param method_id: - :param source_id: - :param qcl_id: - :return: - """ - try: - result = self._edit_session.query(Series).filter_by( - site_id=site_id, - variable_id=var_id, - method_id=method_id, - source_id=source_id, - quality_control_level_id=qcl_id - ).one() - - return True - except: - return False - def qcl_exists(self, q): - """ - - :param q: - :return: - """ - try: - result = self._edit_session.query(QualityControlLevel).filter_by(code=q.code, definition=q.definition).one() - return True - except: - - return False - - def method_exists(self, m): - """ - - :param m: - :return: - """ - try: - result = self._edit_session.query(Method).filter_by(description=m.description).one() - return True - except: - return False - - def variable_exists(self, v): - """ - - :param v: - :return: - """ - try: - result = self._edit_session.query(Variable).filter_by(code=v.code, - name=v.name, speciation=v.speciation, - variable_unit_id=v.variable_unit_id, - sample_medium=v.sample_medium, - value_type=v.value_type, is_regular=v.is_regular, - time_support=v.time_support, - time_unit_id=v.time_unit_id, data_type=v.data_type, - general_category=v.general_category, - no_data_value=v.no_data_value).one() - return result - except: - return None \ No newline at end of file diff --git a/odmtools/odmservices/service_manager.py b/odmtools/odmservices/service_manager.py new file mode 100755 index 0000000..e8159ea --- /dev/null +++ b/odmtools/odmservices/service_manager.py @@ -0,0 +1,208 @@ +import logging +import os +import sys + +import urllib + +from sqlalchemy.exc import SQLAlchemyError +from odmtools.odmservices import SeriesService, CVService, EditService, ExportService +from odmtools.controller import EditTools +from odmtools.lib.Appdirs.appdirs import user_config_dir +from odmtools.odmdata import SessionFactory + + + +class ServiceManager(): + def __init__(self, debug=False): + self.debug = debug + f = self._get_file('r') + self._conn_dicts = [] + self.version = 0 + self._connection_format = "%s+%s://%s:%s@%s/%s" + + # Read all lines (connections) in the connection.cfg file + while True: + line = f.readline() + if not line: + break + else: + line = line.split() + #logger.debug(line) + + if len(line) >= 5: + line_dict = {} + + line_dict['engine'] = line[0] + line_dict['user'] = line[1] + line_dict['password'] = line[2] + line_dict['address'] = line[3] + line_dict['db'] = line[4] + self._conn_dicts.append(line_dict) + + if len(self._conn_dicts) is not 0: + # The current connection defaults to the most recent (i.e. the last written to the file) + self._current_conn_dict = self._conn_dicts[-1] + else: + self._current_conn_dict = None + + f.close() + + def get_all_conn_dicts(self): + return self._conn_dicts + + def is_valid_connection(self): + if self._current_conn_dict: + conn_string = self._build_connection_string(self._current_conn_dict) + logger.debug("Conn_string: %s" % conn_string) + try: + if self.testEngine(conn_string): + return self.get_current_conn_dict() + except Exception as e: + logger.fatal("The previous database for some reason isn't accessible, please enter a new connection %s" % e.message) + return None + return None + + def get_current_conn_dict(self): + return self._current_conn_dict + + def set_current_conn_dict(self, dict): + self._current_conn_dict = dict + + def add_connection(self, conn_dict): + """conn_dict must be a dictionary with keys: engine, user, password, address, db""" + + # remove earlier connections that are identical to this one + self.delete_connection(conn_dict) + + if self.test_connection(conn_dict): + # write changes to connection file + self._conn_dicts.append(conn_dict) + self._current_conn_dict = self._conn_dicts[-1] + self._save_connections() + return True + else: + logger.error("Unable to save connection due to invalid connection to database") + return False + + + @classmethod + def testEngine(self, connection_string): + s = SessionFactory(connection_string, echo=False) + if 'mssql' in connection_string: + s.ms_test_Session().execute("Select top 1 VariableCode From Variables") + elif 'mysql' in connection_string: + s.my_test_Session().execute('Select "VariableCode" From Variables Limit 1') + elif 'postgresql' in connection_string: + #s.psql_test_Session().execute('Select "VariableCode" From "ODM2"."Variables" Limit 1') + s.psql_test_Session().execute('Select "VariableCode" From "Variables" Limit 1') + return True + + def test_connection(self, conn_dict): + try: + conn_string = self._build_connection_string(conn_dict) + if self.testEngine(conn_string) and self.get_db_version(conn_string) == '1.1.1': + return True + except SQLAlchemyError as e: + logger.error("SQLAlchemy Error: %s" % e.message) + raise e + except Exception as e: + logger.error("Error: %s" % e) + raise e + return False + + def delete_connection(self, conn_dict): + self._conn_dicts[:] = [x for x in self._conn_dicts if x != conn_dict] + + # Create and return services based on the currently active connection + def get_db_version_dict(self, conn_dict): + conn_string = self._build_connection_string(conn_dict) + self.get_db_version(conn_string) + + def get_db_version(self, conn_string): + if isinstance(conn_string, dict): + conn_string = self._build_connection_string(conn_string) + service = SeriesService(conn_string) + #if not self.version: + try: + self.version = service.get_db_version() + except Exception as e: + logger.error("Exception: %s" % e.message) + return None + return self.version + + def get_series_service(self, conn_dict=""): + conn_string = "" + if conn_dict: + conn_string = self._build_connection_string(conn_dict) + self._current_conn_dict = conn_dict + else: + conn_string = self._build_connection_string(self._current_conn_dict) + return SeriesService(conn_string, self.debug) + + def get_cv_service(self): + conn_string = self._build_connection_string(self._current_conn_dict) + return CVService(conn_string, self.debug) + + def get_edit_service(self, series_id, connection): + + return EditService(series_id, connection=connection, debug=self.debug) + + def get_record_service(self, script, series_id, connection): + return EditTools(self, script, self.get_edit_service(series_id, connection), + self._build_connection_string(self.is_valid_connection())) + + def get_export_service(self): + return ExportService(self.get_series_service()) + + ## ################### + # private variables + ## ################### + + def _get_file(self, mode): + #fn = util.resource_path('connection.config') + fn = os.path.join(user_config_dir("ODMTools", "UCHIC"), 'connection.config') + + config_file = None + try: + + if os.path.exists(fn): + config_file = open(fn, mode) + else: + os.makedirs(user_config_dir("ODMTools", "UCHIC")) + open(fn, 'w').close() + config_file = open(fn, mode) + except: + open(fn, 'w').close() + config_file = open(fn, mode) + + + return config_file + + def _build_connection_string(self, conn_dict): + driver = "" + if conn_dict['engine'] == 'mssql' and sys.platform != 'win32': + driver = "pyodbc" + quoted = urllib.quote_plus('DRIVER={FreeTDS};DSN=%s;UID=%s;PWD=%s;' % (conn_dict['address'], conn_dict['user'], conn_dict['password'])) + conn_string = 'mssql+pyodbc:///?odbc_connect={}'.format(quoted) + + else: + if conn_dict['engine'] == 'mssql': + driver = "pyodbc" + elif conn_dict['engine'] == 'mysql': + driver = "pymysql" + elif conn_dict['engine'] == 'postgresql': + driver = "psycopg2" + else: + driver = "None" + + conn_string = self._connection_format % ( + conn_dict['engine'], driver, conn_dict['user'], conn_dict['password'], conn_dict['address'], + conn_dict['db']) + return conn_string + + def _save_connections(self): + f = self._get_file('w') + for conn in self._conn_dicts: + f.write("%s %s %s %s %s\n" % (conn['engine'], conn['user'], conn['password'], conn['address'], conn['db'])) + f.close() + From d4e8fef9c01ee1d6848d97762ae519ec4d1a5f51 Mon Sep 17 00:00:00 2001 From: Stephanie Reeder Date: Wed, 26 Aug 2015 13:32:27 -0600 Subject: [PATCH 04/15] update odmdata import --- odmtools/controller/olvSeriesSelector.py | 2 +- odmtools/controller/pageExisting.py | 2 +- odmtools/odmdata/__init__.py | 67 ++++++++++-------------- odmtools/odmdata/memory_database.py | 7 ++- odmtools/odmservices/__init__.py | 5 +- odmtools/odmservices/service_manager.py | 27 +++++----- 6 files changed, 49 insertions(+), 61 deletions(-) diff --git a/odmtools/controller/olvSeriesSelector.py b/odmtools/controller/olvSeriesSelector.py index 7f21f32..0a9a4f1 100644 --- a/odmtools/controller/olvSeriesSelector.py +++ b/odmtools/controller/olvSeriesSelector.py @@ -48,7 +48,7 @@ def _buildColumns(self): ColumnDefn(key, align="left", minimumWidth=100, valueGetter=value, # stringConverter = '%s') stringConverter='%Y-%m-%d %H:%M:%S' if "date" in key.lower() else'%s') - for key, value in series.returnDict().iteritems()] + for key, value in Series.returnDict().iteritems()] self.SetColumns(seriesColumns) """User can select series using the mouse to click on check boxes """ diff --git a/odmtools/controller/pageExisting.py b/odmtools/controller/pageExisting.py index e7c8318..026cedd 100644 --- a/odmtools/controller/pageExisting.py +++ b/odmtools/controller/pageExisting.py @@ -2,7 +2,7 @@ import wx from odmtools.view import clsExisting -from odmtools.odmdata import series +from odmtools.odmdata import Series import wx.wizard as wiz import datetime diff --git a/odmtools/odmdata/__init__.py b/odmtools/odmdata/__init__.py index 447f960..27362a8 100644 --- a/odmtools/odmdata/__init__.py +++ b/odmtools/odmdata/__init__.py @@ -1,35 +1,4 @@ ''' -from base import Base -from censor_code_cv import CensorCodeCV -from data_type_cv import DataTypeCV -from data_value import DataValue -from general_category_cv import GeneralCategoryCV -from iso_metadata import ISOMetadata -from lab_method import LabMethod -from method import Method -from odm_version import ODMVersion -from offset_type import OffsetType -from qualifier import Qualifier -from quality_control_level import QualityControlLevel -from sample import Sample -from sample_medium_cv import SampleMediumCV -from sample_type_cv import SampleTypeCV -from series import Series -from session_factory import SessionFactory -from site import Site -from site_type_cv import SiteTypeCV -from source import Source -from spatial_reference import SpatialReference -from speciation_cv import SpeciationCV -from topic_category_cv import TopicCategoryCV -from unit import Unit -from value_type_cv import ValueTypeCV -from variable import Variable -from variable_name_cv import VariableNameCV -from vertical_datum_cv import VerticalDatumCV - -from series import copy_series -from data_value import copy_data_value __all__ = [ 'Base', @@ -61,22 +30,40 @@ 'VariableNameCV', 'VerticalDatumCV', 'MemoryDatabase', - 'copy_series', - 'copy_data_value' + ] ''' -import sys -from odmtools.odmdata import memory_database - -from src.api.ODMconnection import SessionFactory -from src.api.versionSwitcher.ODM import DataTypeCV, DataValue, GeneralCategoryCV, ISOMetadata, LabMethod, Method, \ - OffsetType,Qualifier,QualityControlLevel,Sample,SampleMediumCV,SampleTypeCV, Series, Site,SiteTypeCV,Source, \ - SpatialReference,SpeciationCV,TopicCategoryCV,Unit,ValueTypeCV,Variable,VerticalDatumCV +from api.ODMconnection import SessionFactory +from api.versionSwitcher import ODM +DataTypeCV=ODM.DataTypeCV +DataValue=ODM.DataValue +# GeneralCategoryCV=ODM.GeneralCategoryCV +ISOMetadata=ODM.ISOMetadata +LabMethod=ODM.LabMethod +Method=ODM.Method +OffsetType=ODM.OffsetType +Qualifier=ODM.Qualifier +QualityControlLevel=ODM.QualityControlLevel +Sample =ODM.Sample +SampledMediumCV= ODM.SampleMediumCV +# SampleTypeCV=ODM.SampleTypeCV +Series=ODM.Series +Site= ODM.Site +SiteType=ODM.SiteTypeCV +Source =ODM.Source +SpatialReferences=ODM.SpatialReference +SpeciationCV=ODM.SpeciationCV +# TopicCategoryCV=ODM.TopicCategoryCV +Unit= ODM.Unit +# ValueTypeCV=ODM.ValueTypeCV +Variable = ODM.Variable +VerticalDatumCV=ODM.VerticalDatumCV +from odmtools.odmdata.memory_database import MemoryDatabase __all__=[ 'SessionFactory', diff --git a/odmtools/odmdata/memory_database.py b/odmtools/odmdata/memory_database.py index 00b1156..8c0a526 100644 --- a/odmtools/odmdata/memory_database.py +++ b/odmtools/odmdata/memory_database.py @@ -4,6 +4,7 @@ from odmtools.common.logger import LoggerTool from odmtools.odmservices import SeriesService +from odmtools.odmservices import ServiceManager from odmtools.odmdata import DataValue tool = LoggerTool() @@ -21,7 +22,8 @@ def __init__(self, taskserver=None): # Series_Service handles remote database self.series_service = None # Memory_service handles in memory database - self.mem_service = SeriesService("sqlite:///:memory:") + sm = ServiceManager() + self.mem_service = sm.get_series_service(conn_string="sqlite:///:memory:") # TODO clean up closing of program # if taskserver is None: #numproc = cpu_count() @@ -31,7 +33,8 @@ def __init__(self, taskserver=None): self.taskserver = taskserver def reset_edit(self): - self.mem_service = SeriesService("sqlite:///:memory:") + sm = ServiceManager() + self.mem_service = sm.get_series_service(conn_string="sqlite:///:memory:") def set_series_service(self, service): self.series_service = service diff --git a/odmtools/odmservices/__init__.py b/odmtools/odmservices/__init__.py index c850d5c..23dd7c6 100644 --- a/odmtools/odmservices/__init__.py +++ b/odmtools/odmservices/__init__.py @@ -8,9 +8,8 @@ # import pymysql # import pyodbc # #import psycopg2 -import sys -sys.path.append('/Users/stephanie/DEV/ODM2PythonAPI/') -from src.api.ODM1_1_1.services import SeriesService, EditService, CVService, ExportService + +from api.ODM1_1_1.services import SeriesService, EditService, CVService, ExportService from service_manager import ServiceManager __all__ = [ diff --git a/odmtools/odmservices/service_manager.py b/odmtools/odmservices/service_manager.py index e8159ea..6348981 100755 --- a/odmtools/odmservices/service_manager.py +++ b/odmtools/odmservices/service_manager.py @@ -8,8 +8,11 @@ from odmtools.odmservices import SeriesService, CVService, EditService, ExportService from odmtools.controller import EditTools from odmtools.lib.Appdirs.appdirs import user_config_dir -from odmtools.odmdata import SessionFactory +from odmtools.odmdata import SessionFactory, Variable +from odmtools.common.logger import LoggerTool +tool = LoggerTool() +logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG) class ServiceManager(): @@ -88,13 +91,9 @@ def add_connection(self, conn_dict): @classmethod def testEngine(self, connection_string): s = SessionFactory(connection_string, echo=False) - if 'mssql' in connection_string: - s.ms_test_Session().execute("Select top 1 VariableCode From Variables") - elif 'mysql' in connection_string: - s.my_test_Session().execute('Select "VariableCode" From Variables Limit 1') - elif 'postgresql' in connection_string: - #s.psql_test_Session().execute('Select "VariableCode" From "ODM2"."Variables" Limit 1') - s.psql_test_Session().execute('Select "VariableCode" From "Variables" Limit 1') + + s.test_Session().query(Variable.code).limit(1).first() + return True def test_connection(self, conn_dict): @@ -130,22 +129,22 @@ def get_db_version(self, conn_string): return None return self.version - def get_series_service(self, conn_dict=""): - conn_string = "" + def get_series_service(self, conn_dict="", conn_string = ""): + if conn_dict: conn_string = self._build_connection_string(conn_dict) self._current_conn_dict = conn_dict - else: + elif not conn_string: conn_string = self._build_connection_string(self._current_conn_dict) - return SeriesService(conn_string, self.debug) + return SeriesService(SessionFactory(conn_string, self.debug)) def get_cv_service(self): conn_string = self._build_connection_string(self._current_conn_dict) - return CVService(conn_string, self.debug) + return CVService(SessionFactory(conn_string, self.debug)) def get_edit_service(self, series_id, connection): - return EditService(series_id, connection=connection, debug=self.debug) + return EditService(series_id, connection, debug=self.debug) def get_record_service(self, script, series_id, connection): return EditTools(self, script, self.get_edit_service(series_id, connection), From f96c12488999b32408c40f5783d46792926cf3c7 Mon Sep 17 00:00:00 2001 From: sreeder Date: Tue, 1 Sep 2015 15:48:55 -0600 Subject: [PATCH 05/15] make adjustments to work with api --- odmtools/controller/logicPlotOptions.py | 3 +- odmtools/controller/olvSeriesSelector.py | 4 +- odmtools/gui/frmODMTools.py | 11 ++- odmtools/odmdata/__init__.py | 5 +- odmtools/odmdata/memory_database.py | 14 +-- odmtools/odmservices/service_manager.py | 114 ++++++++++++++++------- 6 files changed, 101 insertions(+), 50 deletions(-) diff --git a/odmtools/controller/logicPlotOptions.py b/odmtools/controller/logicPlotOptions.py index 9da1f6f..5bf4e31 100644 --- a/odmtools/controller/logicPlotOptions.py +++ b/odmtools/controller/logicPlotOptions.py @@ -219,7 +219,8 @@ def getSeriesById(self, seriesID): series = self.memDB.series_service.get_series_by_id(seriesID) self.memDB.series_service.reset_session() return series - except: + except Exception as e: + print e return None def getSelectedSeries(self, seriesID): diff --git a/odmtools/controller/olvSeriesSelector.py b/odmtools/controller/olvSeriesSelector.py index 0a9a4f1..8002d8d 100644 --- a/odmtools/controller/olvSeriesSelector.py +++ b/odmtools/controller/olvSeriesSelector.py @@ -6,7 +6,7 @@ from odmtools.lib.ObjectListView import FastObjectListView, ColumnDefn from odmtools.common.logger import LoggerTool -from odmtools.odmdata import Series +from odmtools.odmdata import ODM tool = LoggerTool() @@ -48,7 +48,7 @@ def _buildColumns(self): ColumnDefn(key, align="left", minimumWidth=100, valueGetter=value, # stringConverter = '%s') stringConverter='%Y-%m-%d %H:%M:%S' if "date" in key.lower() else'%s') - for key, value in Series.returnDict().iteritems()] + for key, value in ODM.returnDict().iteritems()] self.SetColumns(seriesColumns) """User can select series using the mouse to click on check boxes """ diff --git a/odmtools/gui/frmODMTools.py b/odmtools/gui/frmODMTools.py index 91c3571..c5e9b30 100755 --- a/odmtools/gui/frmODMTools.py +++ b/odmtools/gui/frmODMTools.py @@ -122,11 +122,12 @@ def _init_database(self, quit_if_cancel=True): while True: ## Database connection is valid, therefore proceed through the rest of the program - if self.service_manager.is_valid_connection(): - conn_dict = None + conn_dict = self.service_manager.is_valid_connection() + if conn_dict: + #conn_dict = None - series_service = self.createService() - conn_dict = self.service_manager.get_current_conn_dict() + series_service = self.createService(conn_dict) + #conn_dict = self.service_manager.get_current_conn_dict() if self.servicesValid(series_service): self.service_manager.add_connection(conn_dict) @@ -458,7 +459,7 @@ def createService(self, conn_dict=""): :return: """ - series_service = self.service_manager.get_series_service(conn_dict=conn_dict) + series_service= self.service_manager.get_series_service(conn_dict=conn_dict)#=connection) return series_service def getServiceManager(self): diff --git a/odmtools/odmdata/__init__.py b/odmtools/odmdata/__init__.py index 27362a8..a4d447e 100644 --- a/odmtools/odmdata/__init__.py +++ b/odmtools/odmdata/__init__.py @@ -38,7 +38,8 @@ from api.ODMconnection import SessionFactory -from api.versionSwitcher import ODM +from api.versionSwitcher import ODM, refreshDB +from api.ODM2.models import change_schema DataTypeCV=ODM.DataTypeCV DataValue=ODM.DataValue # GeneralCategoryCV=ODM.GeneralCategoryCV @@ -94,6 +95,8 @@ 'Variable', 'VariableNameCV', 'VerticalDatumCV', + 'refreshDB', + 'change_schema', 'MemoryDatabase', diff --git a/odmtools/odmdata/memory_database.py b/odmtools/odmdata/memory_database.py index 8c0a526..194a632 100644 --- a/odmtools/odmdata/memory_database.py +++ b/odmtools/odmdata/memory_database.py @@ -85,10 +85,10 @@ def getEditDataValuesforGraph(self): return self.mem_service.get_all_plot_values() def commit(self): - self.mem_service._edit_session.commit() + self.mem_service._session.commit() def rollback(self): - self.mem_service._edit_session.rollback() + self.mem_service._session.rollback() # self.mem_service._session_factory.engine.connect().connection.rollback() #self.updateDF() @@ -102,7 +102,7 @@ def update(self, updates): values(DataValue=bindparam('value')) ) - self.mem_service._edit_session.execute(stmt, updates) + self.mem_service._session.execute(stmt, updates) # self.updateDF() @@ -122,7 +122,7 @@ def updateValue(self, ids, operator, value): #break into chunks to get around sqlites restriction. allowing user to send in only 999 arguments at once chunks=self.chunking(ids) for c in chunks: - q=self.mem_service._edit_session.query(DataValue).filter(DataValue.local_date_time.in_(c)) + q=self.mem_service._session.query(DataValue).filter(DataValue.local_date_time.in_(c)) q.update({DataValue.data_value: query}, False) #self.updateDF() @@ -134,7 +134,7 @@ def chunking(self, data): def updateFlag(self, ids, value): chunks=self.chunking(ids) for c in chunks: - self.mem_service._edit_session.query(DataValue).filter(DataValue.local_date_time.in_(c))\ + self.mem_service._session.query(DataValue).filter(DataValue.local_date_time.in_(c))\ .update({DataValue.qualifier_id: value}, False) @@ -163,7 +163,7 @@ def addPoints(self, points): "SiteID": point[10], "VariableID": point[11], "MethodID": point[12], "SourceID": point[13], "QualityControlLevelID": point[14]} - self.mem_service._edit_session.execute(stmt, vals) + self.mem_service._session.execute(stmt, vals) def stopEdit(self): @@ -222,7 +222,7 @@ def changeSeriesIDs(self, var=None, qcl=None, method=None): :return: """ - query = self.mem_service._edit_session.query(DataValue) + query = self.mem_service._session.query(DataValue) if var is not None: logger.debug(var) query.update({DataValue.variable_id: var}) diff --git a/odmtools/odmservices/service_manager.py b/odmtools/odmservices/service_manager.py index 6348981..3e19a47 100755 --- a/odmtools/odmservices/service_manager.py +++ b/odmtools/odmservices/service_manager.py @@ -8,7 +8,8 @@ from odmtools.odmservices import SeriesService, CVService, EditService, ExportService from odmtools.controller import EditTools from odmtools.lib.Appdirs.appdirs import user_config_dir -from odmtools.odmdata import SessionFactory, Variable +from odmtools.odmdata import SessionFactory, Variable, refreshDB, change_schema + from odmtools.common.logger import LoggerTool tool = LoggerTool() @@ -40,6 +41,7 @@ def __init__(self, debug=False): line_dict['password'] = line[2] line_dict['address'] = line[3] line_dict['db'] = line[4] + line_dict['version']=1.1 self._conn_dicts.append(line_dict) if len(self._conn_dicts) is not 0: @@ -53,23 +55,39 @@ def __init__(self, debug=False): def get_all_conn_dicts(self): return self._conn_dicts + # def is_valid_connection(self): + # if self._current_conn_dict: + # conn_string = self._build_connection_string(self._current_conn_dict) + # logger.debug("Conn_string: %s" % conn_string) + # try: + # if self.testEngine(conn_string): + # return self._current_conn_dict + # + # return None + def is_valid_connection(self): if self._current_conn_dict: conn_string = self._build_connection_string(self._current_conn_dict) logger.debug("Conn_string: %s" % conn_string) + dbtype = self._current_conn_dict['version'] + #dbtype =1.1 + refreshDB(dbtype) + try: if self.testEngine(conn_string): - return self.get_current_conn_dict() + return self._current_conn_dict except Exception as e: logger.fatal("The previous database for some reason isn't accessible, please enter a new connection %s" % e.message) return None - return None + + + def get_current_conn_dict(self): return self._current_conn_dict - def set_current_conn_dict(self, dict): - self._current_conn_dict = dict + # def set_current_conn_dict(self, dict): + # self._current_conn_dict = dict def add_connection(self, conn_dict): """conn_dict must be a dictionary with keys: engine, user, password, address, db""" @@ -77,29 +95,55 @@ def add_connection(self, conn_dict): # remove earlier connections that are identical to this one self.delete_connection(conn_dict) - if self.test_connection(conn_dict): - # write changes to connection file - self._conn_dicts.append(conn_dict) - self._current_conn_dict = self._conn_dicts[-1] - self._save_connections() - return True + #assume connection has already been tested + # if self.test_connection(conn_dict): + # write changes to connection file + self._conn_dicts.append(conn_dict) + self._current_conn_dict = self._conn_dicts[-1] + self._save_connections() + return True + # else: + # logger.error("Unable to save connection due to invalid connection to database") + # return False + + + @staticmethod + def _getSchema(engine): + from sqlalchemy.engine import reflection + + insp=reflection.Inspector.from_engine(engine) + + for name in insp.get_schema_names(): + if 'odm2'== name.lower(): + return name else: - logger.error("Unable to save connection due to invalid connection to database") - return False + return insp.default_schema_name + + @classmethod + def _setSchema(self, engine): + + s = self._getSchema(engine) + change_schema(s) + @classmethod def testEngine(self, connection_string): - s = SessionFactory(connection_string, echo=False) - s.test_Session().query(Variable.code).limit(1).first() + s = SessionFactory(connection_string, echo=False) + try: + # s.ms_test_Session().query(Variable1).limit(1).first() + s.test_Session().query(Variable.code).limit(1).first() + except Exception as e: + print "Connection was unsuccessful ", e.message + return False return True def test_connection(self, conn_dict): try: conn_string = self._build_connection_string(conn_dict) - if self.testEngine(conn_string) and self.get_db_version(conn_string) == '1.1.1': + if self.testEngine(conn_string):# and self.get_db_version(conn_string) == '1.1.1': return True except SQLAlchemyError as e: logger.error("SQLAlchemy Error: %s" % e.message) @@ -113,30 +157,32 @@ def delete_connection(self, conn_dict): self._conn_dicts[:] = [x for x in self._conn_dicts if x != conn_dict] # Create and return services based on the currently active connection - def get_db_version_dict(self, conn_dict): - conn_string = self._build_connection_string(conn_dict) - self.get_db_version(conn_string) - - def get_db_version(self, conn_string): - if isinstance(conn_string, dict): - conn_string = self._build_connection_string(conn_string) - service = SeriesService(conn_string) - #if not self.version: - try: - self.version = service.get_db_version() - except Exception as e: - logger.error("Exception: %s" % e.message) - return None - return self.version + # def get_db_version_dict(self, conn_dict): + # conn_string = self._build_connection_string(conn_dict) + # self.get_db_version(conn_string) + # + # def get_db_version(self, conn_string): + # if isinstance(conn_string, dict): + # conn_string = self._build_connection_string(conn_string) + # service = SeriesService(conn_string) + # #if not self.version: + # try: + # self.version = service.get_db_version() + # except Exception as e: + # logger.error("Exception: %s" % e.message) + # return None + # return self.version def get_series_service(self, conn_dict="", conn_string = ""): if conn_dict: conn_string = self._build_connection_string(conn_dict) - self._current_conn_dict = conn_dict - elif not conn_string: + #self._current_conn_dict = conn_dict + elif not conn_dict and not conn_string: conn_string = self._build_connection_string(self._current_conn_dict) - return SeriesService(SessionFactory(conn_string, self.debug)) + sf = SessionFactory(conn_string, self.debug) + ss= SeriesService(sf) + return ss def get_cv_service(self): conn_string = self._build_connection_string(self._current_conn_dict) From a456718a270f4b3ca27b22a2c043cfd4906d999f Mon Sep 17 00:00:00 2001 From: sreeder Date: Wed, 2 Sep 2015 14:48:57 -0600 Subject: [PATCH 06/15] add version number to db config --- odmtools/controller/frmDBConfig.py | 10 +- odmtools/controller/logicPlotOptions.py | 1 - odmtools/controller/pageExisting.py | 6 +- odmtools/gui/frmODMTools.py | 9 +- odmtools/odmdata/__init__.py | 2 + odmtools/odmservices/service_manager.py | 62 ++++---- odmtools/view/clsDBConfig.py | 187 +++++++++++++++++++++--- 7 files changed, 210 insertions(+), 67 deletions(-) diff --git a/odmtools/controller/frmDBConfig.py b/odmtools/controller/frmDBConfig.py index 46ee811..277f1e8 100755 --- a/odmtools/controller/frmDBConfig.py +++ b/odmtools/controller/frmDBConfig.py @@ -92,10 +92,8 @@ def validateInput(self, conn_dict): wx.MessageBox(message, 'Test Connection', wx.OK) else: #TODO add error message if user cannont connect to the database ( not using VPN) but the db is still 1.1.1) - if not (self.service_manager.get_db_version(conn_dict)): - message = "Cannot connect to the database" - else: - message = "This connection is not a 1.1.1 Database" + + message = "Cannot connect to the database" wx.MessageBox(message, 'Error Occurred', wx.OK | wx.ICON_ERROR) return False @@ -108,8 +106,6 @@ def validateInput(self, conn_dict): return True - - # Returns a dictionary of the database values entered in the form def getFieldValues(self): conn_dict = {} @@ -119,6 +115,7 @@ def getFieldValues(self): conn_dict['password'] = self.txtPass.GetValue() conn_dict['address'] = self.txtServer.GetValue() conn_dict['db'] = self.txtDBName.GetValue() + conn_dict['version']= self.cbVersion.GetValue() return conn_dict @@ -128,6 +125,7 @@ def set_field_values(self): self.txtServer.SetValue(conn['address']) self.txtDBName.SetValue(conn['db']) self.txtUser.SetValue(conn['user']) + self.cbVersion.SetValue(conn['version']) for k, v in self.choices.iteritems(): if v == conn['engine']: diff --git a/odmtools/controller/logicPlotOptions.py b/odmtools/controller/logicPlotOptions.py index 5bf4e31..d1ae4e8 100644 --- a/odmtools/controller/logicPlotOptions.py +++ b/odmtools/controller/logicPlotOptions.py @@ -220,7 +220,6 @@ def getSeriesById(self, seriesID): self.memDB.series_service.reset_session() return series except Exception as e: - print e return None def getSelectedSeries(self, seriesID): diff --git a/odmtools/controller/pageExisting.py b/odmtools/controller/pageExisting.py index 026cedd..e8b87a8 100644 --- a/odmtools/controller/pageExisting.py +++ b/odmtools/controller/pageExisting.py @@ -2,7 +2,7 @@ import wx from odmtools.view import clsExisting -from odmtools.odmdata import Series +from odmtools.odmdata import returnDict import wx.wizard as wiz import datetime @@ -44,8 +44,6 @@ def _init_data(self, series_serv, site_id): - - # Handlers for pnlExisting events. def OnOLVItemSelected(self, event): # TODO: Implement OnOLVItemSelected @@ -62,7 +60,7 @@ def initTable(self, dbservice, site_id): seriesColumns = [clsExisting.ColumnDefn(key, align="left", minimumWidth=-1, valueGetter=value, stringConverter= '%Y-%m-%d %H:%M:%S' if 'date' in key.lower() else '%s') - for key, value in series.returnDict().iteritems()] + for key, value in returnDict().iteritems()] self.panel.olvSeriesList.SetColumns(seriesColumns) objects = dbservice.get_series_by_site(site_id= site_id) diff --git a/odmtools/gui/frmODMTools.py b/odmtools/gui/frmODMTools.py index c5e9b30..4ac74be 100755 --- a/odmtools/gui/frmODMTools.py +++ b/odmtools/gui/frmODMTools.py @@ -113,7 +113,7 @@ def _init_s_Items(self, parent): parent.AddWindow(self._ribbon, 0, wx.EXPAND) parent.AddWindow(self.pnlDocking, 85, flag=wx.ALL | wx.EXPAND) - def _init_database(self, quit_if_cancel=True): + def _init_database(self, quit_if_cancel=True, newConnection= ''): logger.debug("Loading Database...") self.service_manager = ServiceManager() @@ -122,7 +122,10 @@ def _init_database(self, quit_if_cancel=True): while True: ## Database connection is valid, therefore proceed through the rest of the program - conn_dict = self.service_manager.is_valid_connection() + if newConnection: + conn_dict= newConnection + else: + conn_dict = self.service_manager.is_valid_connection() if conn_dict: #conn_dict = None @@ -436,7 +439,7 @@ def onChangeDBConn(self, event): newConnection = db_config.panel.getFieldValues() db_config.Destroy() - if self._init_database(quit_if_cancel=False): + if self._init_database(quit_if_cancel=False, newConnection=newConnection): # if editing, stop editing... if self._ribbon.getEditStatus(): self.stopEdit(event=None) diff --git a/odmtools/odmdata/__init__.py b/odmtools/odmdata/__init__.py index a4d447e..29d6252 100644 --- a/odmtools/odmdata/__init__.py +++ b/odmtools/odmdata/__init__.py @@ -63,6 +63,7 @@ # ValueTypeCV=ODM.ValueTypeCV Variable = ODM.Variable VerticalDatumCV=ODM.VerticalDatumCV +returnDict = ODM.returnDict from odmtools.odmdata.memory_database import MemoryDatabase @@ -97,6 +98,7 @@ 'VerticalDatumCV', 'refreshDB', 'change_schema', + 'returnDict', 'MemoryDatabase', diff --git a/odmtools/odmservices/service_manager.py b/odmtools/odmservices/service_manager.py index 3e19a47..a7e8447 100755 --- a/odmtools/odmservices/service_manager.py +++ b/odmtools/odmservices/service_manager.py @@ -41,7 +41,7 @@ def __init__(self, debug=False): line_dict['password'] = line[2] line_dict['address'] = line[3] line_dict['db'] = line[4] - line_dict['version']=1.1 + line_dict['version']='1.1' if len(line) == 5 else line[5] self._conn_dicts.append(line_dict) if len(self._conn_dicts) is not 0: @@ -55,39 +55,12 @@ def __init__(self, debug=False): def get_all_conn_dicts(self): return self._conn_dicts - # def is_valid_connection(self): - # if self._current_conn_dict: - # conn_string = self._build_connection_string(self._current_conn_dict) - # logger.debug("Conn_string: %s" % conn_string) - # try: - # if self.testEngine(conn_string): - # return self._current_conn_dict - # - # return None - - def is_valid_connection(self): - if self._current_conn_dict: - conn_string = self._build_connection_string(self._current_conn_dict) - logger.debug("Conn_string: %s" % conn_string) - dbtype = self._current_conn_dict['version'] - #dbtype =1.1 - refreshDB(dbtype) - - try: - if self.testEngine(conn_string): - return self._current_conn_dict - except Exception as e: - logger.fatal("The previous database for some reason isn't accessible, please enter a new connection %s" % e.message) - return None - - - def get_current_conn_dict(self): return self._current_conn_dict - # def set_current_conn_dict(self, dict): - # self._current_conn_dict = dict + def set_current_conn_dict(self, dict): + self._current_conn_dict = dict def add_connection(self, conn_dict): """conn_dict must be a dictionary with keys: engine, user, password, address, db""" @@ -139,10 +112,37 @@ def testEngine(self, connection_string): print "Connection was unsuccessful ", e.message return False return True + # def is_valid_connection(self): + # if self._current_conn_dict: + # conn_string = self._build_connection_string(self._current_conn_dict) + # logger.debug("Conn_string: %s" % conn_string) + # try: + # if self.testEngine(conn_string): + # return self._current_conn_dict + # + # return None + + def is_valid_connection(self): + if self._current_conn_dict: + conn_string = self._build_connection_string(self._current_conn_dict) + logger.debug("Conn_string: %s" % conn_string) + dbtype = float(self._current_conn_dict['version']) + #dbtype =1.1 + refreshDB(dbtype) + + try: + if self.testEngine(conn_string): + return self._current_conn_dict + except Exception as e: + logger.fatal("The previous database for some reason isn't accessible, please enter a new connection %s" % e.message) + return None def test_connection(self, conn_dict): try: conn_string = self._build_connection_string(conn_dict) + dbtype = float(conn_dict['version']) + #dbtype =1.1 + refreshDB(dbtype) if self.testEngine(conn_string):# and self.get_db_version(conn_string) == '1.1.1': return True except SQLAlchemyError as e: @@ -248,6 +248,6 @@ def _build_connection_string(self, conn_dict): def _save_connections(self): f = self._get_file('w') for conn in self._conn_dicts: - f.write("%s %s %s %s %s\n" % (conn['engine'], conn['user'], conn['password'], conn['address'], conn['db'])) + f.write("%s %s %s %s %s %s\n" % (conn['engine'], conn['user'], conn['password'], conn['address'], conn['db'], conn['version'])) f.close() diff --git a/odmtools/view/clsDBConfig.py b/odmtools/view/clsDBConfig.py index 4d2490e..8553044 100644 --- a/odmtools/view/clsDBConfig.py +++ b/odmtools/view/clsDBConfig.py @@ -1,4 +1,140 @@ -# -*- coding: utf-8 -*- +# # -*- coding: utf-8 -*- +# +# # ########################################################################## +# ## Python code generated with wxFormBuilder (version Jun 5 2014) +# ## http://www.wxformbuilder.org/ +# ## +# ## PLEASE DO "NOT" EDIT THIS FILE! +# ########################################################################### +# +# import wx +# import wx.xrc +# +# ########################################################################### +# ## Class clsDBConfiguration +# ########################################################################### +# +# class clsDBConfiguration(wx.Panel): +# def __init__(self, parent): +# +# wx.Panel.__init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition, size=wx.Size(500, 291), style=wx.SIMPLE_BORDER | wx.TAB_TRAVERSAL) +# +# self.SetMinSize(wx.Size(442, 291)) +# self.SetMaxSize(wx.Size(627, 291)) +# +# formSizer = wx.BoxSizer(wx.VERTICAL) +# +# sbSizer = wx.StaticBoxSizer(wx.StaticBox(self, wx.ID_ANY, u"Database Connection"), wx.VERTICAL) +# +# connectionSizer = wx.FlexGridSizer(0, 2, 0, 15) +# connectionSizer.AddGrowableCol(1) +# connectionSizer.SetFlexibleDirection(wx.VERTICAL) +# connectionSizer.SetNonFlexibleGrowMode(wx.FLEX_GROWMODE_ALL) +# +# self.stConnType = wx.StaticText(self, wx.ID_ANY, u"Connection Type:", wx.DefaultPosition, wx.DefaultSize, +# wx.ALIGN_RIGHT) +# self.stConnType.Wrap(-1) +# connectionSizer.Add(self.stConnType, 0, wx.ALL | wx.EXPAND, 5) +# +# cbDatabaseTypeChoices = [] +# self.cbDatabaseType = wx.ComboBox(self, wx.ID_ANY, u"Combo!", wx.DefaultPosition, wx.DefaultSize, +# cbDatabaseTypeChoices, wx.CB_READONLY) +# connectionSizer.Add(self.cbDatabaseType, 1, wx.ALL | wx.EXPAND, 5) +# +# self.stServer = wx.StaticText(self, wx.ID_ANY, u"Server:", wx.DefaultPosition, wx.DefaultSize, wx.ALIGN_RIGHT) +# self.stServer.Wrap(-1) +# connectionSizer.Add(self.stServer, 0, wx.ALL | wx.EXPAND, 5) +# +# self.txtServer = wx.TextCtrl(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, +# 0 | wx.FULL_REPAINT_ON_RESIZE | wx.SIMPLE_BORDER) +# connectionSizer.Add(self.txtServer, 1, wx.ALL | wx.EXPAND, 5) +# +# self.stDBName = wx.StaticText(self, wx.ID_ANY, u"Database:", wx.DefaultPosition, wx.DefaultSize, wx.ALIGN_RIGHT) +# self.stDBName.Wrap(-1) +# self.stDBName.SetFont(wx.Font(wx.NORMAL_FONT.GetPointSize(), 70, 90, 90, False, wx.EmptyString)) +# +# connectionSizer.Add(self.stDBName, 0, wx.ALL | wx.EXPAND, 5) +# +# self.txtDBName = wx.TextCtrl(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, +# 0 | wx.SIMPLE_BORDER) +# connectionSizer.Add(self.txtDBName, 0, wx.ALL | wx.EXPAND, 5) +# +# self.stUser = wx.StaticText(self, wx.ID_ANY, u"User:", wx.DefaultPosition, wx.DefaultSize, wx.ALIGN_RIGHT) +# self.stUser.Wrap(-1) +# self.stUser.SetFont(wx.Font(wx.NORMAL_FONT.GetPointSize(), 70, 90, 90, False, wx.EmptyString)) +# +# connectionSizer.Add(self.stUser, 0, wx.ALL | wx.EXPAND, 5) +# +# self.txtUser = wx.TextCtrl(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, +# 0 | wx.SIMPLE_BORDER) +# connectionSizer.Add(self.txtUser, 0, wx.ALL | wx.EXPAND, 5) +# +# self.stPass = wx.StaticText(self, wx.ID_ANY, u"Password:", wx.DefaultPosition, wx.DefaultSize, wx.ALIGN_RIGHT) +# self.stPass.Wrap(-1) +# self.stPass.SetFont(wx.Font(wx.NORMAL_FONT.GetPointSize(), 70, 90, 90, False, wx.EmptyString)) +# +# connectionSizer.Add(self.stPass, 0, wx.ALL | wx.EXPAND, 5) +# +# self.txtPass = wx.TextCtrl(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, +# wx.TE_PASSWORD | wx.SIMPLE_BORDER) +# connectionSizer.Add(self.txtPass, 0, wx.ALL | wx.EXPAND, 5) +# +# sbSizer.Add(connectionSizer, 1, wx.EXPAND, 5) +# +# formSizer.Add(sbSizer, 1, wx.EXPAND, 20) +# +# btnSizer = wx.FlexGridSizer(0, 3, 0, 25) +# btnSizer.AddGrowableCol(0) +# btnSizer.AddGrowableCol(1) +# btnSizer.AddGrowableCol(2) +# btnSizer.SetFlexibleDirection(wx.VERTICAL) +# btnSizer.SetNonFlexibleGrowMode(wx.FLEX_GROWMODE_ALL) +# +# self.btnTest = wx.Button(self, wx.ID_ANY, u"Test Connection", wx.DefaultPosition, wx.DefaultSize, 0) +# btnSizer.Add(self.btnTest, 0, wx.ALL | wx.EXPAND, 5) +# +# self.btnSave = wx.Button(self, wx.ID_ANY, u"Save Connection", wx.DefaultPosition, wx.DefaultSize, 0) +# self.btnSave.Enable(False) +# +# btnSizer.Add(self.btnSave, 0, wx.ALL | wx.EXPAND, 5) +# +# self.btnCancel = wx.Button(self, wx.ID_ANY, u"Cancel", wx.DefaultPosition, wx.DefaultSize, 0) +# btnSizer.Add(self.btnCancel, 0, wx.ALL | wx.EXPAND, 5) +# +# formSizer.Add(btnSizer, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.SHAPED, 15) +# +# self.SetSizer(formSizer) +# self.Layout() +# +# # Connect Events +# self.cbDatabaseType.Bind(wx.EVT_COMBOBOX, self.OnValueChanged) +# self.txtServer.Bind(wx.EVT_TEXT, self.OnValueChanged) +# self.txtDBName.Bind(wx.EVT_TEXT, self.OnValueChanged) +# self.txtUser.Bind(wx.EVT_TEXT, self.OnValueChanged) +# self.txtPass.Bind(wx.EVT_TEXT, self.OnValueChanged) +# self.btnTest.Bind(wx.EVT_BUTTON, self.OnBtnTest) +# self.btnSave.Bind(wx.EVT_BUTTON, self.OnBtnSave) +# self.btnCancel.Bind(wx.EVT_BUTTON, self.OnBtnCancel) +# +# def __del__(self): +# pass +# +# +# # Virtual event handlers, overide them in your derived class +# def OnValueChanged(self, event): +# event.Skip() +# +# +# def OnBtnTest(self, event): +# event.Skip() +# +# def OnBtnSave(self, event): +# event.Skip() +# +# def OnBtnCancel(self, event): +# event.Skip() +# +# -*- coding: utf-8 -*- # ########################################################################## ## Python code generated with wxFormBuilder (version Jun 5 2014) @@ -16,7 +152,6 @@ class clsDBConfiguration(wx.Panel): def __init__(self, parent): - wx.Panel.__init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition, size=wx.Size(500, 291), style=wx.SIMPLE_BORDER | wx.TAB_TRAVERSAL) @@ -32,19 +167,30 @@ def __init__(self, parent): connectionSizer.SetFlexibleDirection(wx.VERTICAL) connectionSizer.SetNonFlexibleGrowMode(wx.FLEX_GROWMODE_ALL) + self.stVersion = wx.StaticText(self, wx.ID_ANY, u"DB Version:", wx.DefaultPosition, wx.DefaultSize, + wx.ALIGN_RIGHT) + self.stVersion.Wrap(-1) + connectionSizer.Add(self.stVersion, 0, wx.ALL | wx.ALIGN_RIGHT | wx.EXPAND, 5) + + version_choices = [ u"1.1", u"2.0"] + self.cbVersion = wx.ComboBox(self, wx.ID_ANY, u"1.1", wx.DefaultPosition, wx.DefaultSize, + version_choices, wx.CB_READONLY | wx.CB_SORT) + self.cbVersion.SetSelection(0) + connectionSizer.Add(self.cbVersion, 1, wx.ALL | wx.EXPAND, 5) + self.stConnType = wx.StaticText(self, wx.ID_ANY, u"Connection Type:", wx.DefaultPosition, wx.DefaultSize, wx.ALIGN_RIGHT) self.stConnType.Wrap(-1) - connectionSizer.Add(self.stConnType, 0, wx.ALL | wx.EXPAND, 5) + connectionSizer.Add(self.stConnType, 0, wx.ALL | wx.EXPAND | wx.ALIGN_RIGHT, 5) cbDatabaseTypeChoices = [] - self.cbDatabaseType = wx.ComboBox(self, wx.ID_ANY, u"Combo!", wx.DefaultPosition, wx.DefaultSize, - cbDatabaseTypeChoices, wx.CB_READONLY) + self.cbDatabaseType = wx.ComboBox(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, + cbDatabaseTypeChoices, wx.CB_READONLY | wx.CB_SORT) connectionSizer.Add(self.cbDatabaseType, 1, wx.ALL | wx.EXPAND, 5) self.stServer = wx.StaticText(self, wx.ID_ANY, u"Server:", wx.DefaultPosition, wx.DefaultSize, wx.ALIGN_RIGHT) self.stServer.Wrap(-1) - connectionSizer.Add(self.stServer, 0, wx.ALL | wx.EXPAND, 5) + connectionSizer.Add(self.stServer, 0, wx.ALL | wx.EXPAND | wx.ALIGN_RIGHT, 5) self.txtServer = wx.TextCtrl(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, 0 | wx.FULL_REPAINT_ON_RESIZE | wx.SIMPLE_BORDER) @@ -54,35 +200,35 @@ def __init__(self, parent): self.stDBName.Wrap(-1) self.stDBName.SetFont(wx.Font(wx.NORMAL_FONT.GetPointSize(), 70, 90, 90, False, wx.EmptyString)) - connectionSizer.Add(self.stDBName, 0, wx.ALL | wx.EXPAND, 5) + connectionSizer.Add(self.stDBName, 0, wx.ALL | wx.EXPAND | wx.ALIGN_RIGHT, 5) self.txtDBName = wx.TextCtrl(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, 0 | wx.SIMPLE_BORDER) - connectionSizer.Add(self.txtDBName, 0, wx.ALL | wx.EXPAND, 5) + connectionSizer.Add(self.txtDBName, 1, wx.ALL | wx.EXPAND, 5) self.stUser = wx.StaticText(self, wx.ID_ANY, u"User:", wx.DefaultPosition, wx.DefaultSize, wx.ALIGN_RIGHT) self.stUser.Wrap(-1) self.stUser.SetFont(wx.Font(wx.NORMAL_FONT.GetPointSize(), 70, 90, 90, False, wx.EmptyString)) - connectionSizer.Add(self.stUser, 0, wx.ALL | wx.EXPAND, 5) + connectionSizer.Add(self.stUser, 0, wx.ALL | wx.EXPAND | wx.ALIGN_RIGHT, 5) self.txtUser = wx.TextCtrl(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, 0 | wx.SIMPLE_BORDER) - connectionSizer.Add(self.txtUser, 0, wx.ALL | wx.EXPAND, 5) + connectionSizer.Add(self.txtUser, 1, wx.ALL | wx.EXPAND, 5) self.stPass = wx.StaticText(self, wx.ID_ANY, u"Password:", wx.DefaultPosition, wx.DefaultSize, wx.ALIGN_RIGHT) self.stPass.Wrap(-1) self.stPass.SetFont(wx.Font(wx.NORMAL_FONT.GetPointSize(), 70, 90, 90, False, wx.EmptyString)) - connectionSizer.Add(self.stPass, 0, wx.ALL | wx.EXPAND, 5) + connectionSizer.Add(self.stPass, 0, wx.ALL | wx.EXPAND | wx.ALIGN_RIGHT, 5) self.txtPass = wx.TextCtrl(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, wx.TE_PASSWORD | wx.SIMPLE_BORDER) - connectionSizer.Add(self.txtPass, 0, wx.ALL | wx.EXPAND, 5) + connectionSizer.Add(self.txtPass, 1, wx.ALL | wx.EXPAND, 5) - sbSizer.Add(connectionSizer, 1, wx.EXPAND, 5) + sbSizer.Add(connectionSizer, 90, wx.EXPAND, 3) - formSizer.Add(sbSizer, 1, wx.EXPAND, 20) + formSizer.Add(sbSizer, 1, wx.ALL | wx.EXPAND, 7) btnSizer = wx.FlexGridSizer(0, 3, 0, 25) btnSizer.AddGrowableCol(0) @@ -92,17 +238,15 @@ def __init__(self, parent): btnSizer.SetNonFlexibleGrowMode(wx.FLEX_GROWMODE_ALL) self.btnTest = wx.Button(self, wx.ID_ANY, u"Test Connection", wx.DefaultPosition, wx.DefaultSize, 0) - btnSizer.Add(self.btnTest, 0, wx.ALL | wx.EXPAND, 5) + btnSizer.Add(self.btnTest, 0, wx.ALL | wx.EXPAND | wx.ALIGN_CENTER_VERTICAL | wx.ALIGN_CENTER_HORIZONTAL, 5) self.btnSave = wx.Button(self, wx.ID_ANY, u"Save Connection", wx.DefaultPosition, wx.DefaultSize, 0) - self.btnSave.Enable(False) - - btnSizer.Add(self.btnSave, 0, wx.ALL | wx.EXPAND, 5) + btnSizer.Add(self.btnSave, 0, wx.ALL | wx.EXPAND | wx.ALIGN_CENTER_VERTICAL | wx.ALIGN_CENTER_HORIZONTAL, 5) self.btnCancel = wx.Button(self, wx.ID_ANY, u"Cancel", wx.DefaultPosition, wx.DefaultSize, 0) - btnSizer.Add(self.btnCancel, 0, wx.ALL | wx.EXPAND, 5) + btnSizer.Add(self.btnCancel, 0, wx.ALL | wx.EXPAND | wx.ALIGN_CENTER_VERTICAL | wx.ALIGN_CENTER_HORIZONTAL, 5) - formSizer.Add(btnSizer, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.SHAPED, 15) + formSizer.Add(btnSizer, 10, wx.EXPAND, 2) self.SetSizer(formSizer) self.Layout() @@ -120,12 +264,10 @@ def __init__(self, parent): def __del__(self): pass - # Virtual event handlers, overide them in your derived class def OnValueChanged(self, event): event.Skip() - def OnBtnTest(self, event): event.Skip() @@ -135,3 +277,4 @@ def OnBtnSave(self, event): def OnBtnCancel(self, event): event.Skip() + From 9c6e794058c55b8dfca5b92765288d5d793b71fc Mon Sep 17 00:00:00 2001 From: Stephanie Reeder Date: Thu, 10 Sep 2015 14:51:11 -0600 Subject: [PATCH 07/15] fix errors with ODM2 connection --- odmtools/controller/frmDBConfig.py | 4 +-- odmtools/controller/olvSeriesSelector.py | 4 +-- odmtools/odmdata/__init__.py | 11 ++++-- odmtools/odmdata/memory_database.py | 14 ++++---- odmtools/odmservices/service_manager.py | 34 +++++++++---------- tests/test_odmservices/test_series_service.py | 8 ++--- 6 files changed, 39 insertions(+), 36 deletions(-) diff --git a/odmtools/controller/frmDBConfig.py b/odmtools/controller/frmDBConfig.py index 46ee811..19068f8 100755 --- a/odmtools/controller/frmDBConfig.py +++ b/odmtools/controller/frmDBConfig.py @@ -94,8 +94,8 @@ def validateInput(self, conn_dict): #TODO add error message if user cannont connect to the database ( not using VPN) but the db is still 1.1.1) if not (self.service_manager.get_db_version(conn_dict)): message = "Cannot connect to the database" - else: - message = "This connection is not a 1.1.1 Database" + # else: + # message = "This connection is not a 1.1.1 Database" wx.MessageBox(message, 'Error Occurred', wx.OK | wx.ICON_ERROR) return False diff --git a/odmtools/controller/olvSeriesSelector.py b/odmtools/controller/olvSeriesSelector.py index 0a9a4f1..8002d8d 100644 --- a/odmtools/controller/olvSeriesSelector.py +++ b/odmtools/controller/olvSeriesSelector.py @@ -6,7 +6,7 @@ from odmtools.lib.ObjectListView import FastObjectListView, ColumnDefn from odmtools.common.logger import LoggerTool -from odmtools.odmdata import Series +from odmtools.odmdata import ODM tool = LoggerTool() @@ -48,7 +48,7 @@ def _buildColumns(self): ColumnDefn(key, align="left", minimumWidth=100, valueGetter=value, # stringConverter = '%s') stringConverter='%Y-%m-%d %H:%M:%S' if "date" in key.lower() else'%s') - for key, value in Series.returnDict().iteritems()] + for key, value in ODM.returnDict().iteritems()] self.SetColumns(seriesColumns) """User can select series using the mouse to click on check boxes """ diff --git a/odmtools/odmdata/__init__.py b/odmtools/odmdata/__init__.py index 27362a8..aaf559e 100644 --- a/odmtools/odmdata/__init__.py +++ b/odmtools/odmdata/__init__.py @@ -38,7 +38,9 @@ from api.ODMconnection import SessionFactory -from api.versionSwitcher import ODM +from api.versionSwitcher import ODM, refreshDB +from api.ODM2.models import change_schema + DataTypeCV=ODM.DataTypeCV DataValue=ODM.DataValue # GeneralCategoryCV=ODM.GeneralCategoryCV @@ -62,6 +64,7 @@ # ValueTypeCV=ODM.ValueTypeCV Variable = ODM.Variable VerticalDatumCV=ODM.VerticalDatumCV +returnDict = ODM.returnDict from odmtools.odmdata.memory_database import MemoryDatabase @@ -74,7 +77,7 @@ 'ISOMetadata', 'LabMethod', 'Method', - 'ODMVersion', + #'ODMVersion', 'OffsetType', 'Qualifier', 'QualityControlLevel', @@ -94,7 +97,9 @@ 'Variable', 'VariableNameCV', 'VerticalDatumCV', + 'refreshDB', + 'change_schema', + 'returnDict', 'MemoryDatabase', - ] diff --git a/odmtools/odmdata/memory_database.py b/odmtools/odmdata/memory_database.py index 8c0a526..194a632 100644 --- a/odmtools/odmdata/memory_database.py +++ b/odmtools/odmdata/memory_database.py @@ -85,10 +85,10 @@ def getEditDataValuesforGraph(self): return self.mem_service.get_all_plot_values() def commit(self): - self.mem_service._edit_session.commit() + self.mem_service._session.commit() def rollback(self): - self.mem_service._edit_session.rollback() + self.mem_service._session.rollback() # self.mem_service._session_factory.engine.connect().connection.rollback() #self.updateDF() @@ -102,7 +102,7 @@ def update(self, updates): values(DataValue=bindparam('value')) ) - self.mem_service._edit_session.execute(stmt, updates) + self.mem_service._session.execute(stmt, updates) # self.updateDF() @@ -122,7 +122,7 @@ def updateValue(self, ids, operator, value): #break into chunks to get around sqlites restriction. allowing user to send in only 999 arguments at once chunks=self.chunking(ids) for c in chunks: - q=self.mem_service._edit_session.query(DataValue).filter(DataValue.local_date_time.in_(c)) + q=self.mem_service._session.query(DataValue).filter(DataValue.local_date_time.in_(c)) q.update({DataValue.data_value: query}, False) #self.updateDF() @@ -134,7 +134,7 @@ def chunking(self, data): def updateFlag(self, ids, value): chunks=self.chunking(ids) for c in chunks: - self.mem_service._edit_session.query(DataValue).filter(DataValue.local_date_time.in_(c))\ + self.mem_service._session.query(DataValue).filter(DataValue.local_date_time.in_(c))\ .update({DataValue.qualifier_id: value}, False) @@ -163,7 +163,7 @@ def addPoints(self, points): "SiteID": point[10], "VariableID": point[11], "MethodID": point[12], "SourceID": point[13], "QualityControlLevelID": point[14]} - self.mem_service._edit_session.execute(stmt, vals) + self.mem_service._session.execute(stmt, vals) def stopEdit(self): @@ -222,7 +222,7 @@ def changeSeriesIDs(self, var=None, qcl=None, method=None): :return: """ - query = self.mem_service._edit_session.query(DataValue) + query = self.mem_service._session.query(DataValue) if var is not None: logger.debug(var) query.update({DataValue.variable_id: var}) diff --git a/odmtools/odmservices/service_manager.py b/odmtools/odmservices/service_manager.py index 6348981..e00c2ff 100755 --- a/odmtools/odmservices/service_manager.py +++ b/odmtools/odmservices/service_manager.py @@ -87,11 +87,9 @@ def add_connection(self, conn_dict): logger.error("Unable to save connection due to invalid connection to database") return False - @classmethod def testEngine(self, connection_string): s = SessionFactory(connection_string, echo=False) - s.test_Session().query(Variable.code).limit(1).first() return True @@ -99,7 +97,7 @@ def testEngine(self, connection_string): def test_connection(self, conn_dict): try: conn_string = self._build_connection_string(conn_dict) - if self.testEngine(conn_string) and self.get_db_version(conn_string) == '1.1.1': + if self.testEngine(conn_string): #and self.get_db_version(conn_string) == '1.1.1': return True except SQLAlchemyError as e: logger.error("SQLAlchemy Error: %s" % e.message) @@ -113,21 +111,21 @@ def delete_connection(self, conn_dict): self._conn_dicts[:] = [x for x in self._conn_dicts if x != conn_dict] # Create and return services based on the currently active connection - def get_db_version_dict(self, conn_dict): - conn_string = self._build_connection_string(conn_dict) - self.get_db_version(conn_string) - - def get_db_version(self, conn_string): - if isinstance(conn_string, dict): - conn_string = self._build_connection_string(conn_string) - service = SeriesService(conn_string) - #if not self.version: - try: - self.version = service.get_db_version() - except Exception as e: - logger.error("Exception: %s" % e.message) - return None - return self.version + # def get_db_version_dict(self, conn_dict): + # conn_string = self._build_connection_string(conn_dict) + # self.get_db_version(conn_string) + + # def get_db_version(self, conn_string): + # if isinstance(conn_string, dict): + # conn_string = self._build_connection_string(conn_string) + # service = SeriesService(conn_string) + # #if not self.version: + # try: + # self.version = service.get_db_version() + # except Exception as e: + # logger.error("Exception: %s" % e.message) + # return None + # return self.version def get_series_service(self, conn_dict="", conn_string = ""): diff --git a/tests/test_odmservices/test_series_service.py b/tests/test_odmservices/test_series_service.py index d33f5a5..5f708c3 100644 --- a/tests/test_odmservices/test_series_service.py +++ b/tests/test_odmservices/test_series_service.py @@ -12,10 +12,10 @@ def setup(self): engine = self.series_service._session_factory.engine test_util.build_db(engine) - def test_get_db_version(self): - version = test_util.add_version(self.session) - db_version = self.series_service.get_db_version() - assert version.version_number == db_version + # def test_get_db_version(self): + # version = test_util.add_version(self.session) + # db_version = self.series_service.get_db_version() + # assert version.version_number == db_version def test_get_all_sites_empty(self): sites = self.series_service.get_used_sites() From 0504fe2ef542c94bec41588bfff8d57589716dad Mon Sep 17 00:00:00 2001 From: sreeder Date: Tue, 22 Sep 2015 11:51:45 -0600 Subject: [PATCH 08/15] update code to work with new ODM format --- odmtools/controller/frmCreateVariable.py | 36 +++--- odmtools/controller/olvSeriesSelector.py | 23 +++- odmtools/gui/frmFlagValues.py | 2 +- odmtools/gui/pageMethod.py | 9 +- odmtools/gui/pageQCL.py | 5 +- odmtools/gui/pageVariable.py | 4 +- odmtools/odmdata/__init__.py | 135 ++++++++--------------- odmtools/odmdata/memory_database.py | 30 ++--- odmtools/odmservices/__init__.py | 4 +- odmtools/odmservices/service_manager.py | 12 +- 10 files changed, 119 insertions(+), 141 deletions(-) diff --git a/odmtools/controller/frmCreateVariable.py b/odmtools/controller/frmCreateVariable.py index 5b49153..056cce6 100644 --- a/odmtools/controller/frmCreateVariable.py +++ b/odmtools/controller/frmCreateVariable.py @@ -2,7 +2,7 @@ import wx from odmtools.view.clsCreateVariable import clsCreateVariable -from odmtools.odmdata import Variable +# from odmtools.odmdata import Variable # Implementing clsCreateVariable class frmCreateVariable(clsCreateVariable): @@ -87,26 +87,30 @@ def OnBtnCreateButton(self, event): def createVariable(self): - v = Variable() - v.code = self.txtVarCode.GetValue() if self.txtVarCode.GetValue() <> u'' else None - v.name = self.cbVarName.GetValue() if self.cbVarName.GetValue() <> u'' else None - v.speciation = self.cbSpeciation.GetValue() if self.cbSpeciation.GetValue() <> u'' else None - v.variable_unit = self.series_service.get_unit_by_name( self.cbVarUnits.GetValue()) - v.variable_unit_id = v.variable_unit.id + code = self.txtVarCode.GetValue() if self.txtVarCode.GetValue() <> u'' else None + name = self.cbVarName.GetValue() if self.cbVarName.GetValue() <> u'' else None + speciation = self.cbSpeciation.GetValue() if self.cbSpeciation.GetValue() <> u'' else None - v.sample_medium = self.cbSampleMedium.GetValue() if self.cbSampleMedium.GetValue() <> u'' else None - v.value_type = self.cbValueType.GetValue() if self.cbValueType.GetValue() <> u'' else None - v.is_regular = self.cbIsRegular.GetValue() if self.cbIsRegular.GetValue() <> u'' else None - v.time_support = self.txtTSValue.GetValue() if self.txtTSValue.GetValue() <> u'' else None + variable_unit = self.series_service.get_unit_by_name( self.cbVarUnits.GetValue()) + variable_unit_id = variable_unit.id - v.time_unit = self.series_service.get_unit_by_name(self.cbTSUnits.GetValue()) - v.time_unit_id = v.time_unit.id + sample_medium = self.cbSampleMedium.GetValue() if self.cbSampleMedium.GetValue() <> u'' else None + value_type = self.cbValueType.GetValue() if self.cbValueType.GetValue() <> u'' else None + is_regular = self.cbIsRegular.GetValue() if self.cbIsRegular.GetValue() <> u'' else None + time_support = self.txtTSValue.GetValue() if self.txtTSValue.GetValue() <> u'' else None - v.data_type = self.cbDataType.GetValue() if self.cbDataType.GetValue() <> u'' else None - v.general_category = self.cbGenCat.GetValue() if self.cbGenCat.GetValue() <> u'' else None - v.no_data_value = self.txtNoDV.GetValue() if self.txtNoDV.GetValue() <> u'' else None + time_unit = self.series_service.get_unit_by_name(self.cbTSUnits.GetValue()) + time_unit_id = time_unit.id + + data_type = self.cbDataType.GetValue() if self.cbDataType.GetValue() <> u'' else None + general_category = self.cbGenCat.GetValue() if self.cbGenCat.GetValue() <> u'' else None + no_data_value = self.txtNoDV.GetValue() if self.txtNoDV.GetValue() <> u'' else None + v = self.series_service.create_variable( + code, name, speciation, variable_unit_id, sample_medium, + value_type, is_regular, time_support, time_unit_id, data_type, + general_category, no_data_value) return v def OnBtnCancelButton(self, event): diff --git a/odmtools/controller/olvSeriesSelector.py b/odmtools/controller/olvSeriesSelector.py index 8002d8d..daf92b8 100644 --- a/odmtools/controller/olvSeriesSelector.py +++ b/odmtools/controller/olvSeriesSelector.py @@ -4,16 +4,29 @@ import wx.lib.newevent # from ObjectListView.ObjectListView import FastObjectListView, ColumnDefn from odmtools.lib.ObjectListView import FastObjectListView, ColumnDefn - +# from odmtools.odmdata import returnDict from odmtools.common.logger import LoggerTool -from odmtools.odmdata import ODM - tool = LoggerTool() logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG) OvlCheckEvent, EVT_OVL_CHECK_EVENT = wx.lib.newevent.NewEvent() - +from collections import OrderedDict +def returnDict(): + keys = ['SeriesID', 'SiteID', 'SiteCode', 'SiteName', 'VariableID', 'VariableCode', 'VariableName', 'Speciation', + 'VariableUnitsID', 'VariableUnitsName', 'SampleMedium', 'ValueType', 'TimeSupport', 'TimeUnitsID', + 'TimeUnitsName', 'DataType', 'GeneralCategory', 'MethodID', 'MethodDescription', 'SourceID', + 'SourceDescription', 'Organization', 'Citation', 'QualityControlLevelID', 'QualityControlLevelCode', + 'BeginDateTime', 'EndDateTime', 'BeginDateTimeUTC', 'EndDateTimeUTC', 'ValueCount' + ] + values = ['id', 'site_id', 'site_code', 'site_name', 'variable_id', 'variable_code', 'variable_name', 'speciation', + 'variable_units_id', 'variable_units_name', 'sample_medium', 'value_type', 'time_support', + 'time_units_id', 'time_units_name', 'data_type', 'general_category', 'method_id', 'method_description', + 'source_id', 'source_description', 'organization', 'citation', 'quality_control_level_id', + 'quality_control_level_code', 'begin_date_time', 'end_date_time', 'begin_date_time_utc', + 'end_date_time_utc', 'value_count' + ] + return OrderedDict(zip(keys, values)) class clsSeriesTable(FastObjectListView): def __init__(self, *args, **kwargs): @@ -48,7 +61,7 @@ def _buildColumns(self): ColumnDefn(key, align="left", minimumWidth=100, valueGetter=value, # stringConverter = '%s') stringConverter='%Y-%m-%d %H:%M:%S' if "date" in key.lower() else'%s') - for key, value in ODM.returnDict().iteritems()] + for key, value in returnDict().iteritems()] self.SetColumns(seriesColumns) """User can select series using the mouse to click on check boxes """ diff --git a/odmtools/gui/frmFlagValues.py b/odmtools/gui/frmFlagValues.py index f149a33..e7035f1 100755 --- a/odmtools/gui/frmFlagValues.py +++ b/odmtools/gui/frmFlagValues.py @@ -2,7 +2,7 @@ import wx -from odmtools.odmdata import Qualifier + def create(parent): diff --git a/odmtools/gui/pageMethod.py b/odmtools/gui/pageMethod.py index f422d4a..210770e 100644 --- a/odmtools/gui/pageMethod.py +++ b/odmtools/gui/pageMethod.py @@ -3,7 +3,7 @@ import wx import wx.grid import wx.richtext -from odmtools.odmdata import Method + [wxID_PNLMETHOD, wxID_PNLMETHODSLISTCTRL1, wxID_PNLMETHODSRBCREATENEW, wxID_PNLMETHODSRBGENERATE, wxID_PNLMETHODSRBSELECT, @@ -105,14 +105,15 @@ def OnTxtMethodDescripKillFocus(self, event): def getMethod(self): - m = Method() + m = None if self.rbGenerate.Value: genmethod = "Values derived from ODM Tools Python" try: m= self.series_service.get_method_by_description(genmethod) except: - m.description = genmethod + m =self.series_service(genmethod) + elif self.rbSelect.Value: index = self.lstMethods.GetFirstSelected() @@ -124,5 +125,5 @@ def getMethod(self): elif self.rbCreateNew.Value: - m.description = self.txtMethodDescrip.GetValue() + m =self.series_service( self.txtMethodDescrip.GetValue()) return m \ No newline at end of file diff --git a/odmtools/gui/pageQCL.py b/odmtools/gui/pageQCL.py index af0e411..4c285dc 100644 --- a/odmtools/gui/pageQCL.py +++ b/odmtools/gui/pageQCL.py @@ -1,7 +1,7 @@ #Boa:FramePanel:pnlQCL import wx -from odmtools.odmdata import QualityControlLevel + [wxID_PNLQCL, wxID_PNLQCLLBLCODE, wxID_PNLQCLLBLDEFINITION, wxID_PNLQCLLBLEXPLANATION, wxID_PNLQCLLSTQCL, wxID_PNLQCLRBCREATE, @@ -110,8 +110,9 @@ def GetLstSelection(self): return self.lstQCL.GetFirstSelected() def getQCL(self): - q = QualityControlLevel() + q = None if self.rbCreate.Value: + q= self.series_service.create_qcl(self.txtCode.Value, self.txtDefinition.Value, self.txtExplanation.Value) q.code = self.txtCode.Value q.definition= self.txtDefinition.Value q.explanation = self.txtExplanation.Value diff --git a/odmtools/gui/pageVariable.py b/odmtools/gui/pageVariable.py index 95a7260..e479bbb 100644 --- a/odmtools/gui/pageVariable.py +++ b/odmtools/gui/pageVariable.py @@ -2,7 +2,7 @@ import wx from odmtools.controller.frmCreateVariable import frmCreateVariable -from odmtools.odmdata import Variable +# from odmtools.odmdata import Variable [wxID_PNLVARIABLE, wxID_PNLVARIABLELSTVARIABLE, wxID_PNLVARIABLERBCREATE, wxID_PNLVARIABLERBCURRENT, wxID_PNLVARIABLERBSELECT,wxID_PNLVARIABLETXTNEWVAR, @@ -138,7 +138,7 @@ def OnListCtrl1ListItemSelected(self, event): def getVariable(self): - v = Variable() + v = None if self.rbCurrent.Value: v= self.prev_val elif self.rbSelect.Value: diff --git a/odmtools/odmdata/__init__.py b/odmtools/odmdata/__init__.py index aaf559e..31d31aa 100644 --- a/odmtools/odmdata/__init__.py +++ b/odmtools/odmdata/__init__.py @@ -1,105 +1,64 @@ -''' -__all__ = [ - 'Base', - 'CensorCodeCV', - 'DataTypeCV', - 'DataValue', - 'GeneralCategoryCV', - 'ISOMetadata', - 'LabMethod', - 'Method', - 'ODMVersion', - 'OffsetType', - 'Qualifier', - 'QualityControlLevel', - 'Sample', - 'SampleMediumCV', - 'SampleTypeCV', - 'Series', - 'SessionFactory', - 'Site', - 'SiteTypeCV', - 'Source', - 'SpatialReference', - 'SpeciationCV', - 'TopicCategoryCV', - 'Unit', - 'ValueTypeCV', - 'Variable', - 'VariableNameCV', - 'VerticalDatumCV', - 'MemoryDatabase', -] -''' -from api.ODMconnection import SessionFactory -from api.versionSwitcher import ODM, refreshDB -from api.ODM2.models import change_schema +#from api.versionSwitcher import ODM, refreshDB -DataTypeCV=ODM.DataTypeCV -DataValue=ODM.DataValue -# GeneralCategoryCV=ODM.GeneralCategoryCV -ISOMetadata=ODM.ISOMetadata -LabMethod=ODM.LabMethod -Method=ODM.Method -OffsetType=ODM.OffsetType -Qualifier=ODM.Qualifier -QualityControlLevel=ODM.QualityControlLevel -Sample =ODM.Sample -SampledMediumCV= ODM.SampleMediumCV -# SampleTypeCV=ODM.SampleTypeCV -Series=ODM.Series -Site= ODM.Site -SiteType=ODM.SiteTypeCV -Source =ODM.Source -SpatialReferences=ODM.SpatialReference -SpeciationCV=ODM.SpeciationCV -# TopicCategoryCV=ODM.TopicCategoryCV -Unit= ODM.Unit -# ValueTypeCV=ODM.ValueTypeCV -Variable = ODM.Variable -VerticalDatumCV=ODM.VerticalDatumCV -returnDict = ODM.returnDict +# DataTypeCV=ODM.DataTypeCV +# DataValue=ODM.DataValue +# # GeneralCategoryCV=ODM.GeneralCategoryCV +# ISOMetadata=ODM.ISOMetadata +# LabMethod=ODM.LabMethod +# Method=ODM.Method +# OffsetType=ODM.OffsetType +# Qualifier=ODM.Qualifier +# QualityControlLevel=ODM.QualityControlLevel +# Sample =ODM.Sample +# SampledMediumCV= ODM.SampleMediumCV +# # SampleTypeCV=ODM.SampleTypeCV +# Series=ODM.Series +# Site= ODM.Site +# SiteType=ODM.SiteTypeCV +# Source =ODM.Source +# SpatialReferences=ODM.SpatialReference +# SpeciationCV=ODM.SpeciationCV +# # TopicCategoryCV=ODM.TopicCategoryCV +# Unit= ODM.Unit +# # ValueTypeCV=ODM.ValueTypeCV +# Variable = ODM.Variable +# VerticalDatumCV=ODM.VerticalDatumCV +# returnDict = ODM.returnDict +from api.ODM1_1_1.services.series_service import ODM, refreshDB +from api.ODMconnection import SessionFactory +from api.ODM2.models import change_schema from odmtools.odmdata.memory_database import MemoryDatabase + +from collections import OrderedDict +def returnDict(): + keys = ['SeriesID', 'SiteID', 'SiteCode', 'SiteName', 'VariableID', 'VariableCode', 'VariableName', 'Speciation', + 'VariableUnitsID', 'VariableUnitsName', 'SampleMedium', 'ValueType', 'TimeSupport', 'TimeUnitsID', + 'TimeUnitsName', 'DataType', 'GeneralCategory', 'MethodID', 'MethodDescription', 'SourceID', + 'SourceDescription', 'Organization', 'Citation', 'QualityControlLevelID', 'QualityControlLevelCode', + 'BeginDateTime', 'EndDateTime', 'BeginDateTimeUTC', 'EndDateTimeUTC', 'ValueCount' + ] + values = ['id', 'site_id', 'site_code', 'site_name', 'variable_id', 'variable_code', 'variable_name', 'speciation', + 'variable_units_id', 'variable_units_name', 'sample_medium', 'value_type', 'time_support', + 'time_units_id', 'time_units_name', 'data_type', 'general_category', 'method_id', 'method_description', + 'source_id', 'source_description', 'organization', 'citation', 'quality_control_level_id', + 'quality_control_level_code', 'begin_date_time', 'end_date_time', 'begin_date_time_utc', + 'end_date_time_utc', 'value_count' + ] + return OrderedDict(zip(keys, values)) __all__=[ 'SessionFactory', - 'CensorCodeCV', - 'DataTypeCV', - 'DataValue', - 'GeneralCategoryCV', - 'ISOMetadata', - 'LabMethod', - 'Method', - #'ODMVersion', - 'OffsetType', - 'Qualifier', - 'QualityControlLevel', - 'Sample', - 'SampleMediumCV', - 'SampleTypeCV', - 'Series', - 'SessionFactory', - 'Site', - 'SiteTypeCV', - 'Source', - 'SpatialReference', - 'SpeciationCV', - 'TopicCategoryCV', - 'Unit', - 'ValueTypeCV', - 'Variable', - 'VariableNameCV', - 'VerticalDatumCV', 'refreshDB', 'change_schema', 'returnDict', - + 'ODM', 'MemoryDatabase', + 'returnDict' ] diff --git a/odmtools/odmdata/memory_database.py b/odmtools/odmdata/memory_database.py index 194a632..ceefb98 100644 --- a/odmtools/odmdata/memory_database.py +++ b/odmtools/odmdata/memory_database.py @@ -5,7 +5,7 @@ from odmtools.common.logger import LoggerTool from odmtools.odmservices import SeriesService from odmtools.odmservices import ServiceManager -from odmtools.odmdata import DataValue +from odmtools.odmdata import ODM tool = LoggerTool() logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG) @@ -97,8 +97,8 @@ def update(self, updates): updates : list of dictionary that contains 2 items, id and value ''' - stmt = (DataValue.__table__.update(). - where(DataValue.local_date_time == bindparam('id')). + stmt = (ODM.DataValue.__table__.update(). + where(ODM.DataValue.local_date_time == bindparam('id')). values(DataValue=bindparam('value')) ) @@ -110,11 +110,11 @@ def update(self, updates): def updateValue(self, ids, operator, value): # query = DataValue.data_value+value if operator == '+': - query = DataValue.data_value + value + query = ODM.DataValue.data_value + value elif operator == '-': - query = DataValue.data_value - value + query = ODM.DataValue.data_value - value elif operator == '*': - query = DataValue.data_value * value + query = ODM.DataValue.data_value * value elif operator == '=': query = value @@ -122,8 +122,8 @@ def updateValue(self, ids, operator, value): #break into chunks to get around sqlites restriction. allowing user to send in only 999 arguments at once chunks=self.chunking(ids) for c in chunks: - q=self.mem_service._session.query(DataValue).filter(DataValue.local_date_time.in_(c)) - q.update({DataValue.data_value: query}, False) + q=self.mem_service._session.query(ODM.DataValue).filter(ODM.DataValue.local_date_time.in_(c)) + q.update({ODM.DataValue.data_value: query}, False) #self.updateDF() @@ -134,8 +134,8 @@ def chunking(self, data): def updateFlag(self, ids, value): chunks=self.chunking(ids) for c in chunks: - self.mem_service._session.query(DataValue).filter(DataValue.local_date_time.in_(c))\ - .update({DataValue.qualifier_id: value}, False) + self.mem_service._session.query(ODM.DataValue).filter(ODM.DataValue.local_date_time.in_(c))\ + .update({ODM.DataValue.qualifier_id: value}, False) def delete(self, ids): @@ -149,7 +149,7 @@ def addPoints(self, points): """ Takes in a list of points and loads each point into the database """ - stmt = DataValue.__table__.insert() + stmt = ODM.DataValue.__table__.insert() if not isinstance(points, list): points = [points] @@ -222,17 +222,17 @@ def changeSeriesIDs(self, var=None, qcl=None, method=None): :return: """ - query = self.mem_service._session.query(DataValue) + query = self.mem_service._session.query(ODM.DataValue) if var is not None: logger.debug(var) - query.update({DataValue.variable_id: var}) + query.update({ODM.DataValue.variable_id: var}) if method is not None: logger.debug(method) - query.update({DataValue.method_id: method}) + query.update({ODM.DataValue.method_id: method}) # check that the code is not zero # if qcl is not None and qcl.code != 0: if qcl is not None: logger.debug(qcl) - query.update({DataValue.quality_control_level_id: qcl}) + query.update({ODM.DataValue.quality_control_level_id: qcl}) diff --git a/odmtools/odmservices/__init__.py b/odmtools/odmservices/__init__.py index 23dd7c6..1d39644 100644 --- a/odmtools/odmservices/__init__.py +++ b/odmtools/odmservices/__init__.py @@ -9,12 +9,12 @@ # import pyodbc # #import psycopg2 -from api.ODM1_1_1.services import SeriesService, EditService, CVService, ExportService +from api.ODM1_1_1.services import SeriesService, EditService, ExportService#, , CVService, from service_manager import ServiceManager __all__ = [ 'EditService', - 'CVService', + #'CVService', 'SeriesService', 'ExportService', 'ServiceManager', diff --git a/odmtools/odmservices/service_manager.py b/odmtools/odmservices/service_manager.py index 19c42ba..d287535 100755 --- a/odmtools/odmservices/service_manager.py +++ b/odmtools/odmservices/service_manager.py @@ -5,10 +5,10 @@ import urllib from sqlalchemy.exc import SQLAlchemyError -from odmtools.odmservices import SeriesService, CVService, EditService, ExportService +from odmtools.odmservices import SeriesService, EditService, ExportService from odmtools.controller import EditTools from odmtools.lib.Appdirs.appdirs import user_config_dir -from odmtools.odmdata import SessionFactory, Variable, refreshDB, change_schema +from odmtools.odmdata import SessionFactory, change_schema, refreshDB, ODM from odmtools.common.logger import LoggerTool @@ -105,7 +105,7 @@ def testEngine(self, connection_string): s = SessionFactory(connection_string, echo=False) try: # s.ms_test_Session().query(Variable1).limit(1).first() - s.test_Session().query(Variable.code).limit(1).first() + s.test_Session().query(ODM.Variable.code).limit(1).first() except Exception as e: @@ -186,9 +186,9 @@ def get_series_service(self, conn_dict="", conn_string = ""): ss= SeriesService(sf) return ss - def get_cv_service(self): - conn_string = self._build_connection_string(self._current_conn_dict) - return CVService(SessionFactory(conn_string, self.debug)) + # def get_cv_service(self): + # conn_string = self._build_connection_string(self._current_conn_dict) + # return CVService(SessionFactory(conn_string, self.debug)) def get_edit_service(self, series_id, connection): From 9624eb14875bc389e158220fbf8d422dc8df84cb Mon Sep 17 00:00:00 2001 From: Stephanie Reeder Date: Tue, 29 Sep 2015 09:59:15 -0600 Subject: [PATCH 09/15] update dbversion change --- odmtools/controller/logicPlotOptions.py | 20 ++++++-------------- odmtools/odmdata/__init__.py | 2 +- odmtools/odmdata/memory_database.py | 11 ++++++++--- odmtools/odmservices/service_manager.py | 24 ++++++++++++++---------- odmtools/view/clsDBConfig.py | 4 ++-- 5 files changed, 31 insertions(+), 30 deletions(-) diff --git a/odmtools/controller/logicPlotOptions.py b/odmtools/controller/logicPlotOptions.py index d1ae4e8..460e1a8 100644 --- a/odmtools/controller/logicPlotOptions.py +++ b/odmtools/controller/logicPlotOptions.py @@ -83,7 +83,6 @@ def __init__(self, memDB, taskserver): self.currentEnd = self.endDate self.isSubsetted = False - def getDates(self): return self.startDate, self.endDate, self.currentStart, self.currentEnd @@ -112,7 +111,6 @@ def resetDates(self): self.currentStart = self.startDate self.currentEnd = self.endDate - def isPlotted(self, sid): if int(sid) in self._seriesInfos: return True @@ -144,7 +142,6 @@ def setEditSeries(self, seriesID): self._seriesInfos[self.editID].plotcolor = self._seriesInfos[self.editID].color self._seriesInfos[self.editID].color = "Black" - def updateEditSeries(self): #update values if self.editID in self._seriesInfos: @@ -152,7 +149,6 @@ def updateEditSeries(self): data =self.memDB.getEditDataValuesforGraph() self._seriesInfos[self.editID].dataTable = data - def stopEditSeries(self): if self.editID in self._seriesInfos: data = self.memDB.getDataValuesforGraph( @@ -187,7 +183,6 @@ def update(self, key, isselected): #results = self.taskserver.getCompletedTasks() #self.memDB.setConnection(results["InitEditValues"]) - self._seriesInfos[key] = self.getSeriesInfo(key) self.getUpdatedData(key) @@ -244,6 +239,12 @@ def createSeriesInfo(self, seriesID, seriesInfo, series): unitsName = series.variable_units_name siteName = series.site_name dataType = series.data_type + print type(series) + print dir(series) + print type(series.variable) + print dir(series.variable) + for x in series.variable: + print x noDataValue = series.variable.no_data_value if self.editID == seriesID: #d= DataFrame(pandas.read_sql()) @@ -251,13 +252,11 @@ def createSeriesInfo(self, seriesID, seriesInfo, series): data = self.memDB.getEditDataValuesforGraph() logger.debug("Finished editing -- getting datavalues for graph") - else: logger.debug("plotting -- getting datavalues for graph") data = self.memDB.getDataValuesforGraph(seriesID, noDataValue, self.currentStart, self.currentEnd) logger.debug("Finished plotting -- getting datavalues for graph") - logger.debug("assigning variables...") seriesInfo.seriesID = seriesID seriesInfo.series = series @@ -273,13 +272,11 @@ def createSeriesInfo(self, seriesID, seriesInfo, series): seriesInfo.noDataValue = noDataValue seriesInfo.dataTable = data - if len(data) > 0: seriesInfo.yrange = np.max(data['DataValue']) - np.min(data['DataValue']) else: seriesInfo.yrange = 0 - logger.debug("Finished creating SeriesInfo") return seriesInfo @@ -325,8 +322,6 @@ def buildPlotInfo(self, seriesInfo): seriesInfo.color = self.colorList.pop(0) return seriesInfo - - def updateDateRange(self, startDate=None, endDate=None): self.currentStart = startDate self.currentEnd = endDate @@ -375,7 +370,6 @@ def __init__(self, data): time = timeit.default_timer() - self.NumberofObservations = count self.ArithemticMean = round(np.mean(dvs), 5) self.Maximum = round(np.max(dvs), 5) @@ -403,7 +397,6 @@ def __init__(self, data, method): self.intervals = {} self.method = method - interval_types = ["Overall", "Year", "Month", "Season"] intervals = ["Overall", "Year", "Month", "Season"] @@ -450,7 +443,6 @@ def calculateBoxWhiskerData(self, interval, interval_type): interval_type, interval_type, results["names"], [results["median"], results["conflimit"], results["mean"], results["confint"]]) - def calculateIntervalsOnGroups(self, interval): mean = [] diff --git a/odmtools/odmdata/__init__.py b/odmtools/odmdata/__init__.py index 31d31aa..79e14ac 100644 --- a/odmtools/odmdata/__init__.py +++ b/odmtools/odmdata/__init__.py @@ -31,7 +31,7 @@ # Variable = ODM.Variable # VerticalDatumCV=ODM.VerticalDatumCV # returnDict = ODM.returnDict -from api.ODM1_1_1.services.series_service import ODM, refreshDB +from api.ODM1_1_1.services.series_service import ODM#, refreshDB from api.ODMconnection import SessionFactory from api.ODM2.models import change_schema from odmtools.odmdata.memory_database import MemoryDatabase diff --git a/odmtools/odmdata/memory_database.py b/odmtools/odmdata/memory_database.py index ceefb98..242e0bf 100644 --- a/odmtools/odmdata/memory_database.py +++ b/odmtools/odmdata/memory_database.py @@ -24,6 +24,7 @@ def __init__(self, taskserver=None): # Memory_service handles in memory database sm = ServiceManager() self.mem_service = sm.get_series_service(conn_string="sqlite:///:memory:") + self.mem_service.refreshDB(1.1) # TODO clean up closing of program # if taskserver is None: #numproc = cpu_count() @@ -35,6 +36,7 @@ def __init__(self, taskserver=None): def reset_edit(self): sm = ServiceManager() self.mem_service = sm.get_series_service(conn_string="sqlite:///:memory:") + self.mem_service.refreshDB(1.1) def set_series_service(self, service): self.series_service = service @@ -186,6 +188,8 @@ def updateDF(self): else: ''' self.df = self.mem_service.get_all_values_df() + print self.mem_service._version + def initEditValues(self, seriesID): """ @@ -206,12 +210,13 @@ def initEditValues(self, seriesID): # self.conn = results["InitEditValues"] else: ''' #TODO: Thread this call - if len(self.df)>0: + if self.df is not None and len(self.df)<=0: + logger.debug("no data in series") + else: + self.df.to_sql(name="DataValues", if_exists='replace', con=self.mem_service._session_factory.engine, index=False)#,flavor='sqlite', chunksize=10000) logger.debug("done loading database") - else: - logger.debug("no data in series") def changeSeriesIDs(self, var=None, qcl=None, method=None): """ diff --git a/odmtools/odmservices/service_manager.py b/odmtools/odmservices/service_manager.py index d287535..28d7fa4 100755 --- a/odmtools/odmservices/service_manager.py +++ b/odmtools/odmservices/service_manager.py @@ -8,7 +8,7 @@ from odmtools.odmservices import SeriesService, EditService, ExportService from odmtools.controller import EditTools from odmtools.lib.Appdirs.appdirs import user_config_dir -from odmtools.odmdata import SessionFactory, change_schema, refreshDB, ODM +from odmtools.odmdata import SessionFactory, change_schema, ODM#, refreshDB from odmtools.common.logger import LoggerTool @@ -21,7 +21,7 @@ def __init__(self, debug=False): self.debug = debug f = self._get_file('r') self._conn_dicts = [] - self.version = 0 + #self.version = 0 self._connection_format = "%s+%s://%s:%s@%s/%s" # Read all lines (connections) in the connection.cfg file @@ -41,9 +41,10 @@ def __init__(self, debug=False): line_dict['password'] = line[2] line_dict['address'] = line[3] line_dict['db'] = line[4] - line_dict['version']='1.1' if len(line) == 5 else line[5] + line_dict['version']=1.1 if len(line) == 5 else line[5] self._conn_dicts.append(line_dict) + if len(self._conn_dicts) is not 0: # The current connection defaults to the most recent (i.e. the last written to the file) self._current_conn_dict = self._conn_dicts[-1] @@ -55,7 +56,6 @@ def __init__(self, debug=False): def get_all_conn_dicts(self): return self._conn_dicts - def get_current_conn_dict(self): return self._current_conn_dict @@ -79,7 +79,6 @@ def add_connection(self, conn_dict): # logger.error("Unable to save connection due to invalid connection to database") # return False - @staticmethod def _getSchema(engine): from sqlalchemy.engine import reflection @@ -128,7 +127,7 @@ def is_valid_connection(self): logger.debug("Conn_string: %s" % conn_string) dbtype = float(self._current_conn_dict['version']) #dbtype =1.1 - refreshDB(dbtype) + #refreshDB(dbtype) try: if self.testEngine(conn_string): @@ -143,7 +142,7 @@ def test_connection(self, conn_dict): dbtype = float(conn_dict['version']) #dbtype =1.1 - refreshDB(dbtype) + #refreshDB(dbtype) if self.testEngine(conn_string):# and self.get_db_version(conn_string) == '1.1.1': return True @@ -175,15 +174,20 @@ def delete_connection(self, conn_dict): # return None # return self.version - def get_series_service(self, conn_dict="", conn_string = ""): - + def get_series_service(self, conn_dict=None, conn_string=""): + version = 1.1 if conn_dict: conn_string = self._build_connection_string(conn_dict) #self._current_conn_dict = conn_dict + + version = float(conn_dict['version']) elif not conn_dict and not conn_string: conn_string = self._build_connection_string(self._current_conn_dict) - sf = SessionFactory(conn_string, self.debug) + version = float(self._current_conn_dict['version']) + + sf = SessionFactory(conn_string, self.debug, version = version) ss= SeriesService(sf) + ss.refreshDB(sf.version) return ss # def get_cv_service(self): diff --git a/odmtools/view/clsDBConfig.py b/odmtools/view/clsDBConfig.py index 8553044..3e6231b 100644 --- a/odmtools/view/clsDBConfig.py +++ b/odmtools/view/clsDBConfig.py @@ -174,7 +174,7 @@ def __init__(self, parent): version_choices = [ u"1.1", u"2.0"] self.cbVersion = wx.ComboBox(self, wx.ID_ANY, u"1.1", wx.DefaultPosition, wx.DefaultSize, - version_choices, wx.CB_READONLY | wx.CB_SORT) + version_choices, wx.CB_READONLY )#| wx.CB_SORT) self.cbVersion.SetSelection(0) connectionSizer.Add(self.cbVersion, 1, wx.ALL | wx.EXPAND, 5) @@ -185,7 +185,7 @@ def __init__(self, parent): cbDatabaseTypeChoices = [] self.cbDatabaseType = wx.ComboBox(self, wx.ID_ANY, wx.EmptyString, wx.DefaultPosition, wx.DefaultSize, - cbDatabaseTypeChoices, wx.CB_READONLY | wx.CB_SORT) + cbDatabaseTypeChoices, wx.CB_READONLY )#| wx.CB_SORT) connectionSizer.Add(self.cbDatabaseType, 1, wx.ALL | wx.EXPAND, 5) self.stServer = wx.StaticText(self, wx.ID_ANY, u"Server:", wx.DefaultPosition, wx.DefaultSize, wx.ALIGN_RIGHT) From ae54d9d2f46e801b949cd8d53dbf2313eec784eb Mon Sep 17 00:00:00 2001 From: sreeder Date: Mon, 5 Oct 2015 12:31:23 -0600 Subject: [PATCH 10/15] update code to work with ODM2 api --- odmtools/controller/logicPlotOptions.py | 10 +++------- odmtools/gui/frmODMTools.py | 1 + odmtools/odmdata/memory_database.py | 1 + setup/make.py | 23 +++++++++++++---------- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/odmtools/controller/logicPlotOptions.py b/odmtools/controller/logicPlotOptions.py index 460e1a8..82ef8b7 100644 --- a/odmtools/controller/logicPlotOptions.py +++ b/odmtools/controller/logicPlotOptions.py @@ -239,13 +239,9 @@ def createSeriesInfo(self, seriesID, seriesInfo, series): unitsName = series.variable_units_name siteName = series.site_name dataType = series.data_type - print type(series) - print dir(series) - print type(series.variable) - print dir(series.variable) - for x in series.variable: - print x - noDataValue = series.variable.no_data_value + variable = self.memDB.series_service.get_variable_by_id(series.variable_id) + + noDataValue = variable.no_data_value if self.editID == seriesID: #d= DataFrame(pandas.read_sql()) logger.debug("editing -- getting datavalues for graph") diff --git a/odmtools/gui/frmODMTools.py b/odmtools/gui/frmODMTools.py index 4ac74be..cab5b27 100755 --- a/odmtools/gui/frmODMTools.py +++ b/odmtools/gui/frmODMTools.py @@ -463,6 +463,7 @@ def createService(self, conn_dict=""): """ series_service= self.service_manager.get_series_service(conn_dict=conn_dict)#=connection) + series_service.refreshDB(conn_dict['version']) return series_service def getServiceManager(self): diff --git a/odmtools/odmdata/memory_database.py b/odmtools/odmdata/memory_database.py index 242e0bf..7a8e895 100644 --- a/odmtools/odmdata/memory_database.py +++ b/odmtools/odmdata/memory_database.py @@ -42,6 +42,7 @@ def set_series_service(self, service): self.series_service = service + ############## # DB Queries ############## diff --git a/setup/make.py b/setup/make.py index c1a2cfc..9d23d6f 100644 --- a/setup/make.py +++ b/setup/make.py @@ -160,16 +160,17 @@ def run_pyinstaller(console=False): '--noconfirm ' + APP_FILE) else: ## Non Console Version - val = os.system('pyinstaller ' - '--clean ' - '--distpath=%s ' % WIN_DIR + - '--workpath=%s ' % WORK_DIR + - '--specpath=%s ' % WIN_DIR + - '--upx-dir=%s ' % BASE_DIR + - '--icon=%s ' % WIN_ICON_FILE + - '--version-file=%s ' % VERSION_FILE + - '--noconsole ' - '--noconfirm ' + APP_FILE) + cmd = 'pyinstaller '+\ + '--clean '+\ + '--distpath=%s ' % WIN_DIR +\ + '--workpath=%s ' % WORK_DIR +\ + '--specpath=%s ' % WIN_DIR +\ + '--upx-dir=%s ' % BASE_DIR +\ + '--icon=%s ' % WIN_ICON_FILE +\ + '--version-file=%s ' % VERSION_FILE +\ + '--noconsole '+\ + '--noconfirm ' + APP_FILE + val = os.system(cmd) return True except Exception as e: @@ -237,10 +238,12 @@ def main(): print "Creating Windows Executable..." if run_pyinstaller(): + INNO_SCRIPT = os.path.join(WIN_DIR, "odmtools_no_console.iss") run_inno() print "Creating Windows Executable Console..." if run_pyinstaller(console=True): + INNO_SCRIPT = os.path.join(WIN_DIR, "odmtools_console.iss") run_inno() print "Create No Installer " From f99bf8cbec1e92b4203508d69ed1d866abb6294a Mon Sep 17 00:00:00 2001 From: Stephanie Reeder Date: Tue, 6 Oct 2015 13:44:07 -0600 Subject: [PATCH 11/15] update missing location to handle version switching --- odmtools/gui/frmODMTools.py | 62 ++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/odmtools/gui/frmODMTools.py b/odmtools/gui/frmODMTools.py index cab5b27..2c2be32 100755 --- a/odmtools/gui/frmODMTools.py +++ b/odmtools/gui/frmODMTools.py @@ -31,6 +31,7 @@ tool = LoggerTool() logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG) + class frmODMToolsMain(wx.Frame): """ @@ -50,7 +51,6 @@ def __init__(self, **kwargs): wx.Frame.__init__(self, **kwargs) - series_service = self._init_database() if series_service: self._init_ctrls(series_service) @@ -63,7 +63,6 @@ def __init__(self, **kwargs): logger.debug("System shutting down... ") sys.exit(0) - def _obtainScreenResolution(self): """ Calculates the size of ODMTools. Prevents ODMTools being larger than the available screen size typically a problem on laptops @@ -99,7 +98,6 @@ def _obtainScreenResolution(self): logger.debug("ODMTools Window Size: %s" % newSize) return newSize - #############Entire Form Sizers########## def _init_sizers(self): @@ -113,7 +111,7 @@ def _init_s_Items(self, parent): parent.AddWindow(self._ribbon, 0, wx.EXPAND) parent.AddWindow(self.pnlDocking, 85, flag=wx.ALL | wx.EXPAND) - def _init_database(self, quit_if_cancel=True, newConnection= ''): + def _init_database(self, quit_if_cancel=True, newConnection=''): logger.debug("Loading Database...") self.service_manager = ServiceManager() @@ -123,14 +121,14 @@ def _init_database(self, quit_if_cancel=True, newConnection= ''): while True: ## Database connection is valid, therefore proceed through the rest of the program if newConnection: - conn_dict= newConnection + conn_dict = newConnection else: conn_dict = self.service_manager.is_valid_connection() if conn_dict: - #conn_dict = None + # conn_dict = None series_service = self.createService(conn_dict) - #conn_dict = self.service_manager.get_current_conn_dict() + # conn_dict = self.service_manager.get_current_conn_dict() if self.servicesValid(series_service): self.service_manager.add_connection(conn_dict) @@ -156,7 +154,6 @@ def _init_database(self, quit_if_cancel=True, newConnection= ''): return series_service - def servicesValid(self, service, displayMsg=True): """ @@ -169,13 +166,14 @@ def servicesValid(self, service, displayMsg=True): ## Test if Series Catalog is empty if not service.get_used_sites(): if displayMsg: - msg = wx.MessageDialog(None, 'Series Catalog cannot be empty. Please enter in a new database connection', - 'Series Catalog is empty', wx.OK | wx.ICON_ERROR ) + msg = wx.MessageDialog(None, + 'Series Catalog cannot be empty. Please enter in a new database connection', + 'Series Catalog is empty', wx.OK | wx.ICON_ERROR) msg.ShowModal() valid = False # @TODO If Jeff runs into other issues with services not being available, we can simply test different services here - #if not service.get_all_variables(): + # if not service.get_all_variables(): # valid = False return valid @@ -186,7 +184,7 @@ def on_about_request(self, event): def MacReopenApp(self): """Called when the doc icon is clicked, and ???""" - try: # it's possible for this event to come when the frame is closed + try: # it's possible for this event to come when the frame is closed self.GetTopWindow().Raise() except: pass @@ -230,9 +228,8 @@ def _init_ctrls(self, series_service): ################ Series Selection Panel ################## logger.debug("Loading Series Selector ...") - - self.pnlSelector = FrmSeriesSelector(self.pnlDocking, series_service, plot=self.pnlPlot, taskserver=self.taskserver, memdb = self.memDB) - + self.pnlSelector = FrmSeriesSelector(self.pnlDocking, series_service, plot=self.pnlPlot, + taskserver=self.taskserver, memdb=self.memDB) ####################grid Table View################## logger.debug("Loading DataTable ...") @@ -244,21 +241,18 @@ def _init_ctrls(self, series_service): self.txtPythonConsole = ODMToolsConsole(parent=self.pnlDocking, size=wx.Size(200, 200)) wx.CallAfter(self._postStartup) - - logger.debug("Loading Python Script ...") self.txtPythonScript = pnlScript(name=u'txtPython', parent=self, - size=wx.Size(200, 200)) + size=wx.Size(200, 200)) self.Bind(wx.EVT_CLOSE, self.onClose) - Publisher.subscribe(self.onDocking, ("adjust.Docking")) Publisher.subscribe(self.onPlotSelection, ("select.Plot")) Publisher.subscribe(self.onExecuteScript, ("execute.script")) Publisher.subscribe(self.onChangeDBConn, ("change.dbConfig")) Publisher.subscribe(self.onSetScriptTitle, ("script.title")) - #.subscribe(self.onSetScriptTitle, ("script.title")) + # .subscribe(self.onSetScriptTitle, ("script.title")) Publisher.subscribe(self.onClose, ("onClose")) Publisher.subscribe(self.addEdit, ("selectEdit")) Publisher.subscribe(self.stopEdit, ("stopEdit")) @@ -273,31 +267,31 @@ def _init_aui_manager(self): self._mgr.AddPane(self.pnlPlot, aui.AuiPaneInfo().CenterPane() .Name("Plot").Caption("Plot").MaximizeButton(True).DestroyOnClose(False) - ) + ) self._mgr.AddPane(self.dataTable, aui.AuiPaneInfo().Right().Name("Table"). Show(show=False).Caption('Table View').MinSize(wx.Size(200, 200)).Floatable().Movable(). Position(1).MinimizeButton(True).MaximizeButton(True).DestroyOnClose(False) - ) + ) self._mgr.AddPane(self.pnlSelector, aui.AuiPaneInfo().Bottom().Name("Selector").MinSize(wx.Size(50, 200)). Movable().Floatable().Position(0).MinimizeButton(True).MaximizeButton(True).CloseButton(True) .DestroyOnClose(False) - ) + ) self._mgr.AddPane(self.txtPythonScript, aui.AuiPaneInfo().Caption('Script'). Name("Script").Movable().Floatable().Right() .MinimizeButton(True).MaximizeButton(True).FloatingSize(size=(400, 400)) .CloseButton(True).Float().FloatingPosition(pos=(self.Position)) .Hide().CloseButton(True).DestroyOnClose(False) - ) + ) self._mgr.AddPane(self.txtPythonConsole, aui.AuiPaneInfo().Caption('Python Console'). Name("Console").FloatingSize(size=(300, 400)).MinimizeButton( True).Movable().Floatable().MaximizeButton(True).CloseButton(True).Float() .FloatingPosition(pos=(self.Position)).Show(show=False).DestroyOnClose(False) - ) + ) ## TODO Fix loadingDockingSettings as it doesn't load it correctly. @@ -351,7 +345,6 @@ def getDBService(self): def onPlotSelection(self, value): self.pnlPlot.selectPlot(value) - def onSetScriptTitle(self, title): scriptPane = self._mgr.GetPane(self.txtPythonScript) scriptPane.Caption(title) @@ -361,7 +354,8 @@ def onSetScriptTitle(self, title): def addEdit(self, event): - with wx.BusyInfo("Please wait for a moment while ODMTools fetches the data and stores it in our database", parent=self): + with wx.BusyInfo("Please wait for a moment while ODMTools fetches the data and stores it in our database", + parent=self): logger.debug("Beginning editing") isSelected, seriesID = self.pnlSelector.onReadyToEdit() @@ -407,16 +401,15 @@ def addEdit(self, event): logger.debug("Recording? %s" % self.record_service._record) - #self.record_service = None + # self.record_service = None self.txtPythonConsole.shell.run("edit_service = app.TopWindow.record_service", prompt=False, verbose=False) self.txtPythonConsole.shell.run("series_service = edit_service.get_series_service()", prompt=False, verbose=False) - #from meliae import scanner - #scanner.dump_all_objects("edit_plotting.dat") + # from meliae import scanner + # scanner.dump_all_objects("edit_plotting.dat") logger.info("Finished Setting up Editing Series: %s " % seriesID) - def stopEdit(self, event): self.pnlSelector.stopEdit() @@ -426,7 +419,6 @@ def stopEdit(self, event): self.record_service = None self._ribbon.toggleEditButtons(False) - def getRecordService(self): return self.record_service @@ -444,9 +436,7 @@ def onChangeDBConn(self, event): if self._ribbon.getEditStatus(): self.stopEdit(event=None) - if value == wx.ID_OK: - series_service = self.createService(newConnection) self.pnlSelector.resetDB(series_service) self.refreshConnectionInfo() @@ -462,8 +452,10 @@ def createService(self, conn_dict=""): :return: """ + series_service= self.service_manager.get_series_service(conn_dict=conn_dict)#=connection) series_service.refreshDB(conn_dict['version']) + return series_service def getServiceManager(self): @@ -503,7 +495,7 @@ def onClose(self, event): print "error saving docking data" self._mgr.UnInit() - + # Shut down processes running in background if self.taskserver.numprocesses > 0 and self.taskserver.anyAlive: busy = wx.BusyInfo("Closing ODMTools ...", parent=self) From 9ffdaf102285c3d28d32d475bac66162509d5053 Mon Sep 17 00:00:00 2001 From: Stephanie Reeder Date: Thu, 12 Nov 2015 13:39:40 -0700 Subject: [PATCH 12/15] fix issue printing version # to dbconfig dropdown --- odmtools/controller/frmDBConfig.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/odmtools/controller/frmDBConfig.py b/odmtools/controller/frmDBConfig.py index 6f9b539..0553719 100755 --- a/odmtools/controller/frmDBConfig.py +++ b/odmtools/controller/frmDBConfig.py @@ -127,8 +127,9 @@ def set_field_values(self): self.txtServer.SetValue(conn['address']) self.txtDBName.SetValue(conn['db']) self.txtUser.SetValue(conn['user']) - self.cbVersion.SetValue(conn['version']) + self.cbVersion.SetValue(str(conn['version'])) for k, v in self.choices.iteritems(): if v == conn['engine']: self.cbDatabaseType.SetValue(k) + From 6ebd18ce3a11ff00e832d1dfb8ed714722018370 Mon Sep 17 00:00:00 2001 From: Stephanie Reeder Date: Mon, 10 Oct 2016 15:47:13 -0600 Subject: [PATCH 13/15] update for new api version --- ODMTools.py | 2 +- odmtools/odmdata/__init__.py | 6 +++--- odmtools/odmservices/__init__.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ODMTools.py b/ODMTools.py index 4b2ec5a..5774602 100755 --- a/ODMTools.py +++ b/ODMTools.py @@ -20,7 +20,7 @@ import pyodbc import pymysql -import psycopg2 +#import psycopg2 tool = LoggerTool() logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG) diff --git a/odmtools/odmdata/__init__.py b/odmtools/odmdata/__init__.py index 79e14ac..41deff5 100644 --- a/odmtools/odmdata/__init__.py +++ b/odmtools/odmdata/__init__.py @@ -31,9 +31,9 @@ # Variable = ODM.Variable # VerticalDatumCV=ODM.VerticalDatumCV # returnDict = ODM.returnDict -from api.ODM1_1_1.services.series_service import ODM#, refreshDB -from api.ODMconnection import SessionFactory -from api.ODM2.models import change_schema +from odm2api.ODM1_1_1.services.series_service import ODM#, refreshDB +from odm2api.ODMconnection import SessionFactory +from odm2api.ODM2.models import _changeSchema as change_schema from odmtools.odmdata.memory_database import MemoryDatabase diff --git a/odmtools/odmservices/__init__.py b/odmtools/odmservices/__init__.py index 1d39644..f15d43f 100644 --- a/odmtools/odmservices/__init__.py +++ b/odmtools/odmservices/__init__.py @@ -9,7 +9,7 @@ # import pyodbc # #import psycopg2 -from api.ODM1_1_1.services import SeriesService, EditService, ExportService#, , CVService, +from odm2api.ODM1_1_1.services import SeriesService, EditService, ExportService#, , CVService, from service_manager import ServiceManager __all__ = [ From 1707408725a0a84263aedf5faa69c170a75f3dc6 Mon Sep 17 00:00:00 2001 From: Stephanie Reeder Date: Tue, 11 Oct 2016 15:03:47 -0600 Subject: [PATCH 14/15] update code to use dbconnection from the api --- odmtools/gui/frmODMTools.py | 12 +-- odmtools/odmdata/__init__.py | 12 ++- odmtools/odmdata/memory_database.py | 4 +- odmtools/odmservices/service_manager.py | 125 ++++-------------------- setup/make.py | 2 +- 5 files changed, 35 insertions(+), 120 deletions(-) diff --git a/odmtools/gui/frmODMTools.py b/odmtools/gui/frmODMTools.py index ecea071..8729a35 100755 --- a/odmtools/gui/frmODMTools.py +++ b/odmtools/gui/frmODMTools.py @@ -126,15 +126,11 @@ def _init_database(self, quit_if_cancel=True): while True: ## Database connection is valid, therefore proceed through the rest of the program - if newConnection: - conn_dict = newConnection - else: - conn_dict = self.service_manager.is_valid_connection() - if conn_dict: - # conn_dict = None + if self.service_manager.is_valid_connection(): + conn_dict = None - series_service = self.createService(conn_dict) - # conn_dict = self.service_manager.get_current_conn_dict() + series_service = self.createService() + conn_dict = self.service_manager.get_current_conn_dict() if self.servicesValid(series_service): self.service_manager.add_connection(conn_dict) diff --git a/odmtools/odmdata/__init__.py b/odmtools/odmdata/__init__.py index 41deff5..80b2cba 100644 --- a/odmtools/odmdata/__init__.py +++ b/odmtools/odmdata/__init__.py @@ -31,11 +31,11 @@ # Variable = ODM.Variable # VerticalDatumCV=ODM.VerticalDatumCV # returnDict = ODM.returnDict -from odm2api.ODM1_1_1.services.series_service import ODM#, refreshDB -from odm2api.ODMconnection import SessionFactory +from odm2api.ODM1_1_1.services import SeriesService#, refreshDB +from odm2api.ODMconnection import SessionFactory, dbconnection from odm2api.ODM2.models import _changeSchema as change_schema from odmtools.odmdata.memory_database import MemoryDatabase - +ODM = SeriesService.ODM from collections import OrderedDict def returnDict(): @@ -54,11 +54,13 @@ def returnDict(): ] return OrderedDict(zip(keys, values)) __all__=[ - 'SessionFactory', + #'SessionFactory', 'refreshDB', 'change_schema', 'returnDict', - 'ODM', + #'ODM', 'MemoryDatabase', 'returnDict' + 'SeriesService' + 'dbconnection' ] diff --git a/odmtools/odmdata/memory_database.py b/odmtools/odmdata/memory_database.py index 2490e72..861e32c 100644 --- a/odmtools/odmdata/memory_database.py +++ b/odmtools/odmdata/memory_database.py @@ -5,8 +5,8 @@ from odmtools.common.logger import LoggerTool from odmtools.odmservices import SeriesService from odmtools.odmservices import ServiceManager -from odmtools.odmdata import ODM - +from odmtools.odmdata import SeriesService#ODM +ODM = SeriesService.ODM # tool = LoggerTool() # logger = tool.setupLogger(__name__, __name__ + '.log', 'w', logging.DEBUG) logger =logging.getLogger('main') diff --git a/odmtools/odmservices/service_manager.py b/odmtools/odmservices/service_manager.py index 6fa1b3b..8c9cd7d 100755 --- a/odmtools/odmservices/service_manager.py +++ b/odmtools/odmservices/service_manager.py @@ -8,7 +8,8 @@ from odmtools.odmservices import SeriesService, EditService, ExportService from odmtools.controller import EditTools from odmtools.lib.Appdirs.appdirs import user_config_dir -from odmtools.odmdata import SessionFactory, change_schema, ODM#, refreshDB +from odmtools.odmdata import SeriesService, dbconnection #ODM#, refreshDBSessionFactory, + @@ -44,6 +45,7 @@ def __init__(self, debug=False, conn_dict = None): line_dict['password'] = line[2] line_dict['address'] = line[3] line_dict['db'] = line[4] + line_dict['version']= line[5] self._conn_dicts.append(line_dict) else: self._conn_dicts.append(conn_dict) @@ -84,115 +86,30 @@ def add_connection(self, conn_dict): # logger.error("Unable to save connection due to invalid connection to database") # return False - @staticmethod - def _getSchema(engine): - from sqlalchemy.engine import reflection - - insp=reflection.Inspector.from_engine(engine) - - for name in insp.get_schema_names(): - if 'odm2'== name.lower(): - return name - else: - return insp.default_schema_name - - @classmethod - def _setSchema(self, engine): - - s = self._getSchema(engine) - change_schema(s) - - - @classmethod - def testEngine(self, connection_string): - - s = SessionFactory(connection_string, echo=False) - try: - # s.ms_test_Session().query(Variable1).limit(1).first() - s.test_Session().query(ODM.Variable.code).limit(1).first() - - - except Exception as e: - print "Connection was unsuccessful ", e.message - return False - return True - # def is_valid_connection(self): - # if self._current_conn_dict: - # conn_string = self._build_connection_string(self._current_conn_dict) - # logger.debug("Conn_string: %s" % conn_string) - # try: - # if self.testEngine(conn_string): - # return self._current_conn_dict - # - # return None - - def is_valid_connection(self): - if self._current_conn_dict: - conn_string = self._build_connection_string(self._current_conn_dict) - logger.debug("Conn_string: %s" % conn_string) - dbtype = float(self._current_conn_dict['version']) - #dbtype =1.1 - #refreshDB(dbtype) - - try: - if self.testEngine(conn_string): - return self._current_conn_dict - except Exception as e: - logger.fatal("The previous database for some reason isn't accessible, please enter a new connection %s" % e.message) - return None - - def test_connection(self, conn_dict): - try: - conn_string = self._build_connection_string(conn_dict) - - dbtype = float(conn_dict['version']) - #dbtype =1.1 - #refreshDB(dbtype) - if self.testEngine(conn_string):# and self.get_db_version(conn_string) == '1.1.1': - - return True - except SQLAlchemyError as e: - logger.error("SQLAlchemy Error: %s" % e.message) - raise e - except Exception as e: - logger.error("Error: %s" % e) - raise e - return False def delete_connection(self, conn_dict): self._conn_dicts[:] = [x for x in self._conn_dicts if x != conn_dict] - # Create and return services based on the currently active connection - # def get_db_version_dict(self, conn_dict): - # conn_string = self._build_connection_string(conn_dict) - # self.get_db_version(conn_string) - - # def get_db_version(self, conn_string): - # if isinstance(conn_string, dict): - # conn_string = self._build_connection_string(conn_string) - # service = SeriesService(conn_string) - # #if not self.version: - # try: - # self.version = service.get_db_version() - # except Exception as e: - # logger.error("Exception: %s" % e.message) - # return None - # return self.version def get_series_service(self, conn_dict=None, conn_string=""): - version = 1.1 - if conn_dict: - conn_string = self._build_connection_string(conn_dict) - #self._current_conn_dict = conn_dict - - version = float(conn_dict['version']) - elif not conn_dict and not conn_string: - conn_string = self._build_connection_string(self._current_conn_dict) - version = float(self._current_conn_dict['version']) - - sf = SessionFactory(conn_string, self.debug, version = version) - ss= SeriesService(sf) - ss.refreshDB(sf.version) + if not conn_dict and not conn_string: + conn_dict = self._current_conn_dict + conn = dbconnection.createConnection(conn_dict['engine'], conn_dict['address'], conn_dict['db'], conn_dict['user'], + conn_dict['password'], conn_dict['version']) + + # version = 1.1 + # if conn_dict: + # conn_string = self._build_connection_string(conn_dict) + # #self._current_conn_dict = conn_dict + # + # version = float(conn_dict['version']) + # elif not conn_dict and not conn_string: + # conn_string = self._build_connection_string(self._current_conn_dict) + # version = float(self._current_conn_dict['version']) + # + # sf = SessionFactory(conn_string, self.debug, version = version) + ss= SeriesService(conn) + ss.refreshDB(conn.version) return ss # def get_cv_service(self): diff --git a/setup/make.py b/setup/make.py index 820d0c8..b4ccbf6 100644 --- a/setup/make.py +++ b/setup/make.py @@ -160,7 +160,7 @@ def run_pyinstaller(console=False): '--version-file=%s ' % VERSION_FILE + '--onedir ' # '--onefile ' + - '--exclude= + #'--exclude= '--noconfirm ' + APP_FILE) else: ## Non Console Version From 5bf036c0effdb5baf3aa81adf54cefd675f9096c Mon Sep 17 00:00:00 2001 From: Stephanie Reeder Date: Tue, 11 Oct 2016 17:29:32 -0600 Subject: [PATCH 15/15] get odmtools connecting to odm2 database --- odmtools/gui/frmODMTools.py | 2 +- odmtools/odmdata/memory_database.py | 4 +- odmtools/odmservices/service_manager.py | 83 ++++++++----------------- 3 files changed, 29 insertions(+), 60 deletions(-) diff --git a/odmtools/gui/frmODMTools.py b/odmtools/gui/frmODMTools.py index 8729a35..a153bbd 100755 --- a/odmtools/gui/frmODMTools.py +++ b/odmtools/gui/frmODMTools.py @@ -465,7 +465,7 @@ def createService(self, conn_dict=""): series_service= self.service_manager.get_series_service(conn_dict=conn_dict)#=connection) - series_service.refreshDB(conn_dict['version']) + return series_service diff --git a/odmtools/odmdata/memory_database.py b/odmtools/odmdata/memory_database.py index 861e32c..b10d40a 100644 --- a/odmtools/odmdata/memory_database.py +++ b/odmtools/odmdata/memory_database.py @@ -24,7 +24,7 @@ def __init__(self, taskserver=None): # Memory_service handles in memory database sm = ServiceManager() self.mem_service = sm.get_series_service(conn_string="sqlite:///:memory:") - self.mem_service.refreshDB(1.1) + # TODO clean up closing of program # if taskserver is None: #numproc = cpu_count() @@ -36,7 +36,7 @@ def __init__(self, taskserver=None): def reset_edit(self): sm = ServiceManager() self.mem_service = sm.get_series_service(conn_string="sqlite:///:memory:") - self.mem_service.refreshDB(1.1) + def set_series_service(self, service): self.series_service = service diff --git a/odmtools/odmservices/service_manager.py b/odmtools/odmservices/service_manager.py index 8c9cd7d..a190839 100755 --- a/odmtools/odmservices/service_manager.py +++ b/odmtools/odmservices/service_manager.py @@ -45,7 +45,7 @@ def __init__(self, debug=False, conn_dict = None): line_dict['password'] = line[2] line_dict['address'] = line[3] line_dict['db'] = line[4] - line_dict['version']= line[5] + line_dict['version']= float(line[5]) if len(line)>5 else 1.1 self._conn_dicts.append(line_dict) else: self._conn_dicts.append(conn_dict) @@ -86,17 +86,39 @@ def add_connection(self, conn_dict): # logger.error("Unable to save connection due to invalid connection to database") # return False + def is_valid_connection(self): + + if self.get_current_conn_dict(): + #conn_string = self._build_connection_string(self._current_conn_dict) + #logger.debug("Conn_string: %s" % conn_string) + conn_dict = self.get_current_conn_dict() + try: + if dbconnection.isValidConnection(dbconnection.buildConnectionString(conn_dict['engine'], conn_dict['address'], conn_dict['db'], conn_dict['user'], + conn_dict['password']), dbtype = conn_dict['version']): + return self.get_current_conn_dict() + except Exception as e: + logger.fatal( + "The previous database for some reason isn't accessible, please enter a new connection %s" % e.message) + return None + return None + def delete_connection(self, conn_dict): self._conn_dicts[:] = [x for x in self._conn_dicts if x != conn_dict] def get_series_service(self, conn_dict=None, conn_string=""): - if not conn_dict and not conn_string: - conn_dict = self._current_conn_dict - conn = dbconnection.createConnection(conn_dict['engine'], conn_dict['address'], conn_dict['db'], conn_dict['user'], + if not conn_dict: + conn_dict = self.get_current_conn_dict() + + if conn_string: + #todo how to get version from a connection string + conn = dbconnection.createConnectionFromString(conn_string, float(self.get_current_conn_dict()["version"])) + else: + conn = dbconnection.createConnection(conn_dict['engine'], conn_dict['address'], conn_dict['db'], conn_dict['user'], conn_dict['password'], conn_dict['version']) + # version = 1.1 # if conn_dict: # conn_string = self._build_connection_string(conn_dict) @@ -117,7 +139,6 @@ def get_series_service(self, conn_dict=None, conn_string=""): # return CVService(SessionFactory(conn_string, self.debug)) def get_edit_service(self, series_id, connection): - return EditService(series_id, connection=connection, debug=self.debug) @@ -152,58 +173,6 @@ def _get_file(self, mode): return config_file - def _build_connection_string(self, conn_dict): - - self._connection_format = "%s+%s://%s:%s@%s/%s" - - if conn_dict['engine'] == 'mssql' and sys.platform != 'win32': - driver = "pyodbc" - quoted = urllib.quote_plus('DRIVER={FreeTDS};DSN=%s;UID=%s;PWD=%s;' % (conn_dict['address'], conn_dict['user'], - conn_dict['password'])) - # quoted = urllib.quote_plus('DRIVER={FreeTDS};DSN=%s;UID=%s;PWD=%s;DATABASE=%s' % - # (conn_dict['address'], conn_dict['user'], conn_dict['password'],conn_dict['db'], - # )) - conn_string = 'mssql+pyodbc:///?odbc_connect={}'.format(quoted) - - elif conn_dict['engine']=='sqlite': - connformat = "%s:///%s" - conn_string = connformat%(conn_dict['engine'], conn_dict['address']) - else: - if conn_dict['engine'] == 'mssql': - driver = "pyodbc" - conn = "%s+%s://%s:%s@%s/%s?driver=SQL+Server" - if "sqlncli11.dll" in os.listdir("C:\\Windows\\System32"): - conn = "%s+%s://%s:%s@%s/%s?driver=SQL+Server+Native+Client+11.0" - self._connection_format = conn - conn_string = self._connection_format % ( - conn_dict['engine'], driver, conn_dict['user'], conn_dict['password'], conn_dict['address'], - conn_dict['db']) - elif conn_dict['engine'] == 'mysql': - driver = "pymysql" - conn_string = self.constringBuilder(conn_dict, driver) - elif conn_dict['engine'] == 'postgresql': - driver = "psycopg2" - conn_string = self.constringBuilder(conn_dict, driver) - else: - driver = "None" - conn_string = self.constringBuilder(conn_dict, driver) - - - # print "******", conn_string - return conn_string - - - - def constringBuilder(self, conn_dict, driver): - if conn_dict['password'] is None or not conn_dict['password']: - conn_string = self._connection_format_nopassword % ( - conn_dict['engine'], driver, conn_dict['user'], conn_dict['address'], - conn_dict['db']) - else: - conn_string = self._connection_format % ( - conn_dict['engine'], driver, conn_dict['user'], conn_dict['password'], conn_dict['address'], - conn_dict['db']) - return conn_string def _save_connections(self): f = self._get_file('w')