From 5474a03c73da969a578bac90b6d15d20911b2079 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sat, 8 Jan 2022 13:47:56 +0100 Subject: [PATCH 01/15] Add variable mapping of psm3 --- pvlib/iotools/psm3.py | 58 +++++++++++++++++++++++++++++--- pvlib/tests/iotools/test_psm3.py | 43 ++++++++++++++++++----- 2 files changed, 88 insertions(+), 13 deletions(-) diff --git a/pvlib/iotools/psm3.py b/pvlib/iotools/psm3.py index 44fba674b1..788ed92992 100644 --- a/pvlib/iotools/psm3.py +++ b/pvlib/iotools/psm3.py @@ -8,6 +8,8 @@ import requests import pandas as pd from json import JSONDecodeError +import warnings +from pvlib._deprecation import pvlibDeprecationWarning NSRDB_API_BASE = "https://developer.nrel.gov" PSM_URL = NSRDB_API_BASE + "/api/nsrdb/v2/solar/psm3-download.csv" @@ -20,10 +22,30 @@ 'surface_pressure', 'wind_direction', 'wind_speed') PVLIB_PYTHON = 'pvlib python' +# Dictionary mapping PSM3 names to pvlib names +PSM3_VARIABLE_MAP = { + 'GHI': 'ghi', + 'DHI': 'dhi', + 'DNI': 'dni', + 'Clearsky GHI': 'ghi_clear', + 'Clearsky DHI': 'dhi_clear', + 'Clearsky DNI': 'dni_clear', + 'Solar Zenith Angle': 'solar_zenith', + 'Temperature': 'temp_air', + 'Relative Humidity': 'relative_humidity', + 'Dew point': 'temp_dew', + 'Pressure': 'pressure', + 'Wind Direction': 'wind_direction', + 'Wind Speed': 'wind_speed', + 'Latitude': 'latitude', + 'Longitude': 'longitude', + 'Elevation': 'elevation' + } + def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, attributes=ATTRIBUTES, leap_day=False, full_name=PVLIB_PYTHON, - affiliation=PVLIB_PYTHON, timeout=30): + affiliation=PVLIB_PYTHON, map_variables=None, timeout=30): """ Retrieve NSRDB PSM3 timeseries weather data from the PSM3 API. The NSRDB is described in [1]_ and the PSM3 API is described in [2]_, [3]_, and [4]_. @@ -61,6 +83,9 @@ def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, optional affiliation : str, default 'pvlib python' optional + map_variables: bool + When true, renames columns of the Dataframe to pvlib variable names + where applicable. See variable PSM3_VARIABLE_MAP. timeout : int, default 30 time in seconds to wait for server response before timeout @@ -133,6 +158,13 @@ def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, # convert to string to accomodate integer years being passed in names = str(names) + # convert pvlib names in attributes to psm3 convention (reverse mapping) + # unlike psm3 columns, attributes are lower case and with underscores + amap = {value : key.lower().replace(' ','_') for (key, value) in + PSM3_VARIABLE_MAP.items()} + attributes = [a if a not in amap.keys() else amap[a] for a in attributes] + attributes = list(set(attributes)) # remove duplicate values + # required query-string parameters for request to PSM3 API params = { 'api_key': api_key, @@ -167,10 +199,10 @@ def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, # the CSV is in the response content as a UTF-8 bytestring # to use pandas we need to create a file buffer from the response fbuf = io.StringIO(response.content.decode('utf-8')) - return parse_psm3(fbuf) + return parse_psm3(fbuf, map_variables) -def parse_psm3(fbuf): +def parse_psm3(fbuf, map_variables): """ Parse an NSRDB PSM3 weather file (formatted as SAM CSV). The NSRDB is described in [1]_ and the SAM CSV format is described in [2]_. @@ -184,6 +216,9 @@ def parse_psm3(fbuf): ---------- fbuf: file-like object File-like object containing data to read. + map_variables: bool + When true, renames columns of the Dataframe to pvlib variable names + where applicable. See variable PSM3_VARIABLE_MAP. Returns ------- @@ -296,10 +331,20 @@ def parse_psm3(fbuf): tz = 'Etc/GMT%+d' % -metadata['Time Zone'] data.index = pd.DatetimeIndex(dtidx).tz_localize(tz) + if map_variables is None: + warnings.warn( + 'PSM3 variable names will be renamed to pvlib conventions by ' + 'default starting in pvlib 0.11.0. Specify map_variables=True ' + 'to enable that behavior now, or specify map_variables=False ' + 'to hide this warning.', pvlibDeprecationWarning) + map_variables = False + if map_variables: + data = data.rename(columns=PSM3_VARIABLE_MAP) + return data, metadata -def read_psm3(filename): +def read_psm3(filename, map_variables=None): """ Read an NSRDB PSM3 weather file (formatted as SAM CSV). The NSRDB is described in [1]_ and the SAM CSV format is described in [2]_. @@ -313,6 +358,9 @@ def read_psm3(filename): ---------- filename: str Filename of a file containing data to read. + map_variables: bool + When true, renames columns of the Dataframe to pvlib variable names + where applicable. See variable PSM3_VARIABLE_MAP. Returns ------- @@ -334,5 +382,5 @@ def read_psm3(filename): `_ """ with open(str(filename), 'r') as fbuf: - content = parse_psm3(fbuf) + content = parse_psm3(fbuf, map_variables) return content diff --git a/pvlib/tests/iotools/test_psm3.py b/pvlib/tests/iotools/test_psm3.py index 0b02c3c291..13261eccb4 100644 --- a/pvlib/tests/iotools/test_psm3.py +++ b/pvlib/tests/iotools/test_psm3.py @@ -4,7 +4,7 @@ import os from pvlib.iotools import psm3 -from ..conftest import DATA_DIR, RERUNS, RERUNS_DELAY +from ..conftest import DATA_DIR, RERUNS, RERUNS_DELAY, assert_index_equal import numpy as np import pandas as pd import pytest @@ -76,7 +76,8 @@ def assert_psm3_equal(data, metadata, expected): def test_get_psm3_tmy(nrel_api_key): """test get_psm3 with a TMY""" data, metadata = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key, - PVLIB_EMAIL, names='tmy-2017') + PVLIB_EMAIL, names='tmy-2017', + map_variables=False) expected = pd.read_csv(TMY_TEST_DATA) assert_psm3_equal(data, metadata, expected) @@ -86,7 +87,8 @@ def test_get_psm3_tmy(nrel_api_key): def test_get_psm3_singleyear(nrel_api_key): """test get_psm3 with a single year""" data, metadata = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key, - PVLIB_EMAIL, names='2017', interval=30) + PVLIB_EMAIL, names='2017', + map_variables=False, interval=30) expected = pd.read_csv(YEAR_TEST_DATA) assert_psm3_equal(data, metadata, expected) @@ -96,7 +98,8 @@ def test_get_psm3_singleyear(nrel_api_key): def test_get_psm3_5min(nrel_api_key): """test get_psm3 for 5-minute data""" data, metadata = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key, - PVLIB_EMAIL, names='2019', interval=5) + PVLIB_EMAIL, names='2019', interval=5, + map_variables=False) assert len(data) == 525600/5 first_day = data.loc['2019-01-01'] expected = pd.read_csv(YEAR_TEST_DATA_5MIN) @@ -108,7 +111,7 @@ def test_get_psm3_5min(nrel_api_key): def test_get_psm3_check_leap_day(nrel_api_key): data_2012, _ = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key, PVLIB_EMAIL, names="2012", interval=60, - leap_day=True) + leap_day=True, map_variables=False) assert len(data_2012) == (8760 + 24) @@ -133,7 +136,7 @@ def test_get_psm3_tmy_errors( """ with pytest.raises(HTTPError) as excinfo: psm3.get_psm3(latitude, longitude, api_key, PVLIB_EMAIL, - names=names, interval=interval) + names=names, interval=interval, map_variables=False) # ensure the HTTPError caught isn't due to overuse of the API key assert "OVER_RATE_LIMIT" not in str(excinfo.value) @@ -149,13 +152,37 @@ def io_input(request): def test_parse_psm3(io_input): """test parse_psm3""" - data, metadata = psm3.parse_psm3(io_input) + data, metadata = psm3.parse_psm3(io_input, map_variables=False) expected = pd.read_csv(YEAR_TEST_DATA) assert_psm3_equal(data, metadata, expected) def test_read_psm3(): """test read_psm3""" - data, metadata = psm3.read_psm3(MANUAL_TEST_DATA) + data, metadata = psm3.read_psm3(MANUAL_TEST_DATA, map_variables=False) expected = pd.read_csv(YEAR_TEST_DATA) assert_psm3_equal(data, metadata, expected) + + +def test_read_psm3_map_variables(): + """test read_psm3 map_variables=True""" + data, metadata = psm3.read_psm3(MANUAL_TEST_DATA, map_variables=True) + columns_mapped = ['Year', 'Month', 'Day', 'Hour', 'Minute', 'dhi', 'dni', + 'ghi', 'dhi_clear', 'dni_clear', 'ghi_clear', + 'Cloud Type', 'Dew Point', 'solar_zenith', 'Fill Flag', + 'Surface Albedo', 'wind_speed', 'Precipitable Water', + 'wind_direction', 'relative_humidity', 'temp_air', + 'pressure'] + data, metadata = psm3.read_psm3(MANUAL_TEST_DATA, map_variables=True) + assert_index_equal(data.columns, pd.Index(columns_mapped)) + + +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_psm3_attribute_mapping(nrel_api_key): + """Test that pvlib names can be passed in as attributes and get correctly + reverse mapped to PSM3 names""" + data, _ = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key, PVLIB_EMAIL, + names=2019, interval=60, attributes=['ghi'], + map_variables=True) + assert 'ghi' in data.columns From 04a1fb22a1d6d73984eedf1904b033a8c224f1b1 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sat, 8 Jan 2022 13:55:14 +0100 Subject: [PATCH 02/15] Add enhancement entry in whatsnew --- docs/sphinx/source/whatsnew/v0.9.1.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/sphinx/source/whatsnew/v0.9.1.rst b/docs/sphinx/source/whatsnew/v0.9.1.rst index 64fc94cfa1..8082a1fdfe 100644 --- a/docs/sphinx/source/whatsnew/v0.9.1.rst +++ b/docs/sphinx/source/whatsnew/v0.9.1.rst @@ -11,6 +11,8 @@ Deprecations Enhancements ~~~~~~~~~~~~ +* Added ``map_variables`` option to :py:func:`pvlib.iotools.get_psm3` and + :py:func:`pvlib.iotools.read_psm3` (:pull:`1374`) Bug fixes ~~~~~~~~~ From 80ac77954e1c30854389ce66b945e509e20c92ee Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sat, 8 Jan 2022 13:56:52 +0100 Subject: [PATCH 03/15] Fix stickler --- pvlib/iotools/psm3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/psm3.py b/pvlib/iotools/psm3.py index 788ed92992..40f3611a10 100644 --- a/pvlib/iotools/psm3.py +++ b/pvlib/iotools/psm3.py @@ -40,7 +40,7 @@ 'Latitude': 'latitude', 'Longitude': 'longitude', 'Elevation': 'elevation' - } +} def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, @@ -160,7 +160,7 @@ def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, # convert pvlib names in attributes to psm3 convention (reverse mapping) # unlike psm3 columns, attributes are lower case and with underscores - amap = {value : key.lower().replace(' ','_') for (key, value) in + amap = {value: key.lower().replace(' ', '_') for (key, value) in PSM3_VARIABLE_MAP.items()} attributes = [a if a not in amap.keys() else amap[a] for a in attributes] attributes = list(set(attributes)) # remove duplicate values From 9692da8645532eb5455739e347b1e51fcc08ae8d Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 9 Jan 2022 02:18:06 +0100 Subject: [PATCH 04/15] Map keys in metadata dict --- pvlib/iotools/psm3.py | 3 +++ pvlib/tests/iotools/test_psm3.py | 11 ++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pvlib/iotools/psm3.py b/pvlib/iotools/psm3.py index 40f3611a10..f6d3db10f2 100644 --- a/pvlib/iotools/psm3.py +++ b/pvlib/iotools/psm3.py @@ -340,6 +340,9 @@ def parse_psm3(fbuf, map_variables): map_variables = False if map_variables: data = data.rename(columns=PSM3_VARIABLE_MAP) + metadata['latitude'] = metadata.pop('Latitude') + metadata['longitude'] = metadata.pop('Longitude') + metadata['elevation'] = metadata.pop('Elevation') return data, metadata diff --git a/pvlib/tests/iotools/test_psm3.py b/pvlib/tests/iotools/test_psm3.py index 13261eccb4..f1477b09c7 100644 --- a/pvlib/tests/iotools/test_psm3.py +++ b/pvlib/tests/iotools/test_psm3.py @@ -182,7 +182,12 @@ def test_read_psm3_map_variables(): def test_get_psm3_attribute_mapping(nrel_api_key): """Test that pvlib names can be passed in as attributes and get correctly reverse mapped to PSM3 names""" - data, _ = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key, PVLIB_EMAIL, - names=2019, interval=60, attributes=['ghi'], - map_variables=True) + data, meta = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key, PVLIB_EMAIL, + names=2019, interval=60, + attributes=['ghi', 'wind_speed'], + map_variables=True) assert 'ghi' in data.columns + assert 'wind_speed' in data.columns + assert 'latitude' in meta.keys() + assert 'longitude' in meta.keys() + assert 'elevation' in meta.keys() From c96387bff81ad933d76b3e28c13159519bc98405 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 9 Jan 2022 02:40:52 +0100 Subject: [PATCH 05/15] Remove double spaces in docs --- pvlib/iotools/psm3.py | 15 +++++++-------- pvlib/tests/iotools/test_psm3.py | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/pvlib/iotools/psm3.py b/pvlib/iotools/psm3.py index f6d3db10f2..99091d2a0f 100644 --- a/pvlib/iotools/psm3.py +++ b/pvlib/iotools/psm3.py @@ -1,4 +1,3 @@ - """ Get PSM3 TMY see https://developer.nrel.gov/docs/solar/nsrdb/psm3_data_download/ @@ -30,7 +29,7 @@ 'Clearsky GHI': 'ghi_clear', 'Clearsky DHI': 'dhi_clear', 'Clearsky DNI': 'dni_clear', - 'Solar Zenith Angle': 'solar_zenith', + 'Solar Zenith Angle': 'apparent_zenith', 'Temperature': 'temp_air', 'Relative Humidity': 'relative_humidity', 'Dew point': 'temp_dew', @@ -47,7 +46,7 @@ def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, attributes=ATTRIBUTES, leap_day=False, full_name=PVLIB_PYTHON, affiliation=PVLIB_PYTHON, map_variables=None, timeout=30): """ - Retrieve NSRDB PSM3 timeseries weather data from the PSM3 API. The NSRDB + Retrieve NSRDB PSM3 timeseries weather data from the PSM3 API. The NSRDB is described in [1]_ and the PSM3 API is described in [2]_, [3]_, and [4]_. .. versionchanged:: 0.9.0 @@ -70,14 +69,14 @@ def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, PSM3 API parameter specifing year or TMY variant to download, see notes below for options interval : int, {60, 5, 15, 30} - interval size in minutes, must be 5, 15, 30 or 60. Only used for + interval size in minutes, must be 5, 15, 30 or 60. Only used for single-year requests (i.e., it is ignored for tmy/tgy/tdy requests). attributes : list of str, optional meteorological fields to fetch. If not specified, defaults to ``pvlib.iotools.psm3.ATTRIBUTES``. See references [2]_, [3]_, and [4]_ for lists of available fields. leap_day : boolean, default False - include leap day in the results. Only used for single-year requests + include leap day in the results. Only used for single-year requests (i.e., it is ignored for tmy/tgy/tdy requests). full_name : str, default 'pvlib python' optional @@ -128,7 +127,7 @@ def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, +-----------+-------------------------------------------------------------+ .. warning:: PSM3 is limited to data found in the NSRDB, please consult the - references below for locations with available data. Additionally, + references below for locations with available data. Additionally, querying data with < 30-minute resolution uses a different API endpoint with fewer available fields (see [4]_). @@ -204,7 +203,7 @@ def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, def parse_psm3(fbuf, map_variables): """ - Parse an NSRDB PSM3 weather file (formatted as SAM CSV). The NSRDB + Parse an NSRDB PSM3 weather file (formatted as SAM CSV). The NSRDB is described in [1]_ and the SAM CSV format is described in [2]_. .. versionchanged:: 0.9.0 @@ -349,7 +348,7 @@ def parse_psm3(fbuf, map_variables): def read_psm3(filename, map_variables=None): """ - Read an NSRDB PSM3 weather file (formatted as SAM CSV). The NSRDB + Read an NSRDB PSM3 weather file (formatted as SAM CSV). The NSRDB is described in [1]_ and the SAM CSV format is described in [2]_. .. versionchanged:: 0.9.0 diff --git a/pvlib/tests/iotools/test_psm3.py b/pvlib/tests/iotools/test_psm3.py index f1477b09c7..c6c9250d37 100644 --- a/pvlib/tests/iotools/test_psm3.py +++ b/pvlib/tests/iotools/test_psm3.py @@ -169,7 +169,7 @@ def test_read_psm3_map_variables(): data, metadata = psm3.read_psm3(MANUAL_TEST_DATA, map_variables=True) columns_mapped = ['Year', 'Month', 'Day', 'Hour', 'Minute', 'dhi', 'dni', 'ghi', 'dhi_clear', 'dni_clear', 'ghi_clear', - 'Cloud Type', 'Dew Point', 'solar_zenith', 'Fill Flag', + 'Cloud Type', 'Dew Point', 'apparent_zenith', 'Fill Flag', 'Surface Albedo', 'wind_speed', 'Precipitable Water', 'wind_direction', 'relative_humidity', 'temp_air', 'pressure'] From 12f42e843526b4ef087575b36d473a13b72053b7 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 9 Jan 2022 02:41:46 +0100 Subject: [PATCH 06/15] Fix stickler --- pvlib/tests/iotools/test_psm3.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pvlib/tests/iotools/test_psm3.py b/pvlib/tests/iotools/test_psm3.py index c6c9250d37..730cc3cd6b 100644 --- a/pvlib/tests/iotools/test_psm3.py +++ b/pvlib/tests/iotools/test_psm3.py @@ -169,10 +169,10 @@ def test_read_psm3_map_variables(): data, metadata = psm3.read_psm3(MANUAL_TEST_DATA, map_variables=True) columns_mapped = ['Year', 'Month', 'Day', 'Hour', 'Minute', 'dhi', 'dni', 'ghi', 'dhi_clear', 'dni_clear', 'ghi_clear', - 'Cloud Type', 'Dew Point', 'apparent_zenith', 'Fill Flag', - 'Surface Albedo', 'wind_speed', 'Precipitable Water', - 'wind_direction', 'relative_humidity', 'temp_air', - 'pressure'] + 'Cloud Type', 'Dew Point', 'apparent_zenith', + 'Fill Flag', 'Surface Albedo', 'wind_speed', + 'Precipitable Water', 'wind_direction', + 'relative_humidity', 'temp_air', 'pressure'] data, metadata = psm3.read_psm3(MANUAL_TEST_DATA, map_variables=True) assert_index_equal(data.columns, pd.Index(columns_mapped)) From 4b749bc7583274868e39036e61ac2b172c6d8164 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Tue, 11 Jan 2022 00:49:12 +0100 Subject: [PATCH 07/15] Doc update Co-authored-by: Kevin Anderson <57452607+kanderso-nrel@users.noreply.github.com> --- pvlib/iotools/psm3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/psm3.py b/pvlib/iotools/psm3.py index 99091d2a0f..7f90e95d38 100644 --- a/pvlib/iotools/psm3.py +++ b/pvlib/iotools/psm3.py @@ -82,9 +82,9 @@ def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, optional affiliation : str, default 'pvlib python' optional - map_variables: bool + map_variables: boolean, optional When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable PSM3_VARIABLE_MAP. + where applicable. See variable :const:`PSM3_VARIABLE_MAP`. timeout : int, default 30 time in seconds to wait for server response before timeout From b6d58b69425f07f3d35f335bc249d93ec0d23f52 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 11 Jan 2022 00:52:19 +0100 Subject: [PATCH 08/15] Reformatting - changes by kanderso-nrel --- pvlib/iotools/psm3.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pvlib/iotools/psm3.py b/pvlib/iotools/psm3.py index 7f90e95d38..77d9225cd0 100644 --- a/pvlib/iotools/psm3.py +++ b/pvlib/iotools/psm3.py @@ -36,9 +36,8 @@ 'Pressure': 'pressure', 'Wind Direction': 'wind_direction', 'Wind Speed': 'wind_speed', - 'Latitude': 'latitude', - 'Longitude': 'longitude', - 'Elevation': 'elevation' + 'Surface Albedo': 'albedo', + 'Precipitable Water': 'precipitable_water', } @@ -161,7 +160,7 @@ def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, # unlike psm3 columns, attributes are lower case and with underscores amap = {value: key.lower().replace(' ', '_') for (key, value) in PSM3_VARIABLE_MAP.items()} - attributes = [a if a not in amap.keys() else amap[a] for a in attributes] + attributes = [amap.get(a, a) for a in attributes] attributes = list(set(attributes)) # remove duplicate values # required query-string parameters for request to PSM3 API @@ -201,7 +200,7 @@ def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, return parse_psm3(fbuf, map_variables) -def parse_psm3(fbuf, map_variables): +def parse_psm3(fbuf, map_variables=None): """ Parse an NSRDB PSM3 weather file (formatted as SAM CSV). The NSRDB is described in [1]_ and the SAM CSV format is described in [2]_. From fb74ba73053835480ba5351b131f19d6c48c76d8 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 11 Jan 2022 00:55:06 +0100 Subject: [PATCH 09/15] Update docstring table with 2020 --- pvlib/iotools/psm3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pvlib/iotools/psm3.py b/pvlib/iotools/psm3.py index 77d9225cd0..41924a7208 100644 --- a/pvlib/iotools/psm3.py +++ b/pvlib/iotools/psm3.py @@ -119,10 +119,11 @@ def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, +===========+=============================================================+ | Year | 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, | | | 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, | - | | 2018, 2019 | + | | 2018, 2019, 2020 | +-----------+-------------------------------------------------------------+ | TMY | tmy, tmy-2016, tmy-2017, tdy-2017, tgy-2017, | | | tmy-2018, tdy-2018, tgy-2018, tmy-2019, tdy-2019, tgy-2019 | + | | tmy-2020, tdy-2020, tgy-2020 | +-----------+-------------------------------------------------------------+ .. warning:: PSM3 is limited to data found in the NSRDB, please consult the From e77535852c2ca55f2a46724166ff9ffff213b4bc Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 11 Jan 2022 01:28:28 +0100 Subject: [PATCH 10/15] Add deprecation warning test coverage --- pvlib/tests/iotools/test_psm3.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pvlib/tests/iotools/test_psm3.py b/pvlib/tests/iotools/test_psm3.py index 730cc3cd6b..df914c375d 100644 --- a/pvlib/tests/iotools/test_psm3.py +++ b/pvlib/tests/iotools/test_psm3.py @@ -11,6 +11,7 @@ from requests import HTTPError from io import StringIO import warnings +from pvlib._deprecation import pvlibDeprecationWarning TMY_TEST_DATA = DATA_DIR / 'test_psm3_tmy-2017.csv' YEAR_TEST_DATA = DATA_DIR / 'test_psm3_2017.csv' @@ -170,8 +171,8 @@ def test_read_psm3_map_variables(): columns_mapped = ['Year', 'Month', 'Day', 'Hour', 'Minute', 'dhi', 'dni', 'ghi', 'dhi_clear', 'dni_clear', 'ghi_clear', 'Cloud Type', 'Dew Point', 'apparent_zenith', - 'Fill Flag', 'Surface Albedo', 'wind_speed', - 'Precipitable Water', 'wind_direction', + 'Fill Flag', 'albedo', 'wind_speed', + 'precipitable_water', 'wind_direction', 'relative_humidity', 'temp_air', 'pressure'] data, metadata = psm3.read_psm3(MANUAL_TEST_DATA, map_variables=True) assert_index_equal(data.columns, pd.Index(columns_mapped)) @@ -191,3 +192,10 @@ def test_get_psm3_attribute_mapping(nrel_api_key): assert 'latitude' in meta.keys() assert 'longitude' in meta.keys() assert 'elevation' in meta.keys() + + +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_psm3_variable_map_deprecation_warning(nrel_api_key): + with pytest.warns(pvlibDeprecationWarning, match='names will be renamed'): + _ = psm3.read_psm3(MANUAL_TEST_DATA) From cebb770e17ca0e1358266cece47961a602f23bd6 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Wed, 16 Feb 2022 23:09:42 +0100 Subject: [PATCH 11/15] Rename to VARIABLE_MAP --- pvlib/iotools/psm3.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pvlib/iotools/psm3.py b/pvlib/iotools/psm3.py index 1137f43199..009520d108 100644 --- a/pvlib/iotools/psm3.py +++ b/pvlib/iotools/psm3.py @@ -22,7 +22,7 @@ PVLIB_PYTHON = 'pvlib python' # Dictionary mapping PSM3 names to pvlib names -PSM3_VARIABLE_MAP = { +VARIABLE_MAP = { 'GHI': 'ghi', 'DHI': 'dhi', 'DNI': 'dni', @@ -83,7 +83,7 @@ def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, optional map_variables: boolean, optional When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable :const:`PSM3_VARIABLE_MAP`. + where applicable. See variable :const:`VARIABLE_MAP`. timeout : int, default 30 time in seconds to wait for server response before timeout @@ -160,7 +160,7 @@ def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, # convert pvlib names in attributes to psm3 convention (reverse mapping) # unlike psm3 columns, attributes are lower case and with underscores amap = {value: key.lower().replace(' ', '_') for (key, value) in - PSM3_VARIABLE_MAP.items()} + VARIABLE_MAP.items()} attributes = [amap.get(a, a) for a in attributes] attributes = list(set(attributes)) # remove duplicate values @@ -217,7 +217,7 @@ def parse_psm3(fbuf, map_variables=None): File-like object containing data to read. map_variables: bool When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable PSM3_VARIABLE_MAP. + where applicable. See variable VARIABLE_MAP. Returns ------- @@ -338,7 +338,7 @@ def parse_psm3(fbuf, map_variables=None): 'to hide this warning.', pvlibDeprecationWarning) map_variables = False if map_variables: - data = data.rename(columns=PSM3_VARIABLE_MAP) + data = data.rename(columns=VARIABLE_MAP) metadata['latitude'] = metadata.pop('Latitude') metadata['longitude'] = metadata.pop('Longitude') metadata['elevation'] = metadata.pop('Elevation') @@ -362,7 +362,7 @@ def read_psm3(filename, map_variables=None): Filename of a file containing data to read. map_variables: bool When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable PSM3_VARIABLE_MAP. + where applicable. See variable VARIABLE_MAP. Returns ------- From 9bad5099815618958d8a63d1830c6ff24a5f791e Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Wed, 2 Mar 2022 22:20:32 +0100 Subject: [PATCH 12/15] Change apparent_zenith to solar_zenith Based on the decision in #1403 --- pvlib/iotools/psm3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/psm3.py b/pvlib/iotools/psm3.py index 009520d108..07e48c6f16 100644 --- a/pvlib/iotools/psm3.py +++ b/pvlib/iotools/psm3.py @@ -29,7 +29,7 @@ 'Clearsky GHI': 'ghi_clear', 'Clearsky DHI': 'dhi_clear', 'Clearsky DNI': 'dni_clear', - 'Solar Zenith Angle': 'apparent_zenith', + 'Solar Zenith Angle': 'solar_zenith', 'Temperature': 'temp_air', 'Relative Humidity': 'relative_humidity', 'Dew point': 'temp_dew', From ad2e9de7daf2c87ba1a231a3e17c3321b3a3b1e1 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Wed, 2 Mar 2022 22:24:18 +0100 Subject: [PATCH 13/15] Update attributes docstring --- pvlib/iotools/psm3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pvlib/iotools/psm3.py b/pvlib/iotools/psm3.py index 07e48c6f16..d6d313e875 100644 --- a/pvlib/iotools/psm3.py +++ b/pvlib/iotools/psm3.py @@ -73,7 +73,8 @@ def get_psm3(latitude, longitude, api_key, email, names='tmy', interval=60, attributes : list of str, optional meteorological fields to fetch. If not specified, defaults to ``pvlib.iotools.psm3.ATTRIBUTES``. See references [2]_, [3]_, and [4]_ - for lists of available fields. + for lists of available fields. Alternatively, pvlib names may also be + used (e.g. 'ghi' rather than 'GHI'); see :const:`VARIABLE_MAP`. leap_day : boolean, default False include leap day in the results. Only used for single-year requests (i.e., it is ignored for tmy/tgy/tdy requests). From 82af8be60cc398d46ba2642b8f25b90bfc2e9ca3 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Thu, 3 Mar 2022 20:03:52 +0100 Subject: [PATCH 14/15] Change elevation to altitude when mapping variables --- pvlib/iotools/psm3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/psm3.py b/pvlib/iotools/psm3.py index 009520d108..9d9703d362 100644 --- a/pvlib/iotools/psm3.py +++ b/pvlib/iotools/psm3.py @@ -341,7 +341,7 @@ def parse_psm3(fbuf, map_variables=None): data = data.rename(columns=VARIABLE_MAP) metadata['latitude'] = metadata.pop('Latitude') metadata['longitude'] = metadata.pop('Longitude') - metadata['elevation'] = metadata.pop('Elevation') + metadata['altitude'] = metadata.pop('Elevation') return data, metadata From 7033c944544cc436db9f1d63b666a2921cfbc723 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Thu, 3 Mar 2022 20:20:02 +0100 Subject: [PATCH 15/15] Update psm3 variable mapping test --- pvlib/tests/iotools/test_psm3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/tests/iotools/test_psm3.py b/pvlib/tests/iotools/test_psm3.py index b7dc07bb19..2c20155cdd 100644 --- a/pvlib/tests/iotools/test_psm3.py +++ b/pvlib/tests/iotools/test_psm3.py @@ -191,7 +191,7 @@ def test_get_psm3_attribute_mapping(nrel_api_key): assert 'wind_speed' in data.columns assert 'latitude' in meta.keys() assert 'longitude' in meta.keys() - assert 'elevation' in meta.keys() + assert 'altitude' in meta.keys() @pytest.mark.remote_data