From cb6c4cb40ef8b0bae4452b3f85bd34507518a132 Mon Sep 17 00:00:00 2001 From: DresdenGman Date: Tue, 3 Nov 2026 15:16:09 +0000 Subject: [PATCH 1/2] Move _parse_bsrn and _parse_epw functionality into read_bsrn and read_epw --- docs/sphinx/source/whatsnew/v0.16.2.rst | 5 + pvlib/iotools/bsrn.py | 283 +++++++++++------------- pvlib/iotools/epw.py | 132 ++++------- 3 files changed, 186 insertions(+), 234 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.16.2.rst b/docs/sphinx/source/whatsnew/v0.16.2.rst index 5893d8bf46..d88a8145a9 100644 --- a/docs/sphinx/source/whatsnew/v0.16.2.rst +++ b/docs/sphinx/source/whatsnew/v0.16.2.rst @@ -65,6 +65,10 @@ Requirements Maintenance ~~~~~~~~~~~ +* Move the parsing functionality of the private ``_parse_bsrn`` and + ``_parse_epw`` functions into :py:func:`~pvlib.iotools.read_bsrn` and + :py:func:`~pvlib.iotools.read_epw`, and remove the private functions. + (:issue:`2469`) Contributors @@ -80,3 +84,4 @@ Contributors * :ghuser:`bfs2021` * :ghuser:`xiajiahang` * Jakub Krasuski (:ghuser:`Blizzeq`) +* Dresden Goehner (:ghuser:`DresdenGman`) diff --git a/pvlib/iotools/bsrn.py b/pvlib/iotools/bsrn.py index 7c02866e1c..e0e27d92ad 100644 --- a/pvlib/iotools/bsrn.py +++ b/pvlib/iotools/bsrn.py @@ -191,7 +191,7 @@ def get_bsrn(station, start, end, username, password, bio.seek(0) # reset buffer to start of file gzip_file = io.TextIOWrapper(gzip.GzipFile(fileobj=bio), encoding='latin1') - dfi, metadata = _parse_bsrn(gzip_file, logical_records) + dfi, metadata = read_bsrn(gzip_file, logical_records) dfs.append(dfi) # FTP client raises an error if the file does not exist on server except ftplib.error_perm as e: @@ -217,153 +217,6 @@ def get_bsrn(station, start, end, username, password, return data, metadata -def _parse_bsrn(fbuf, logical_records=('0100',)): - """ - Parse a file-like buffer of a BSRN station-to-archive file. - - Parameters - ---------- - fbuf: file-like buffer - Buffer of a BSRN station-to-archive data file - logical_records: list or tuple, default: ('0100',) - List of the logical records (LR) to parse. Options include: '0100', - '0300', and '0500'. - - Returns - ------- - data: DataFrame - timeseries data from the BSRN archive, see - :func:`pvlib.iotools.read_bsrn` for fields. An empty DataFrame is - returned if the specified logical records were not found. - metadata: dict - Dictionary containing metadata (primarily from LR0004). - - See Also - -------- - pvlib.iotools.read_bsrn, pvlib.iotools.get_bsrn - - """ - # Parse metadata - fbuf.readline() # first line should be *U0001, so read it and discard - date_line = fbuf.readline() # second line contains important metadata - start_date = pd.Timestamp(year=int(date_line[7:11]), - month=int(date_line[3:6]), day=1, - tz='UTC') # BSRN timestamps are UTC - - metadata = {} # Initilize dictionary containing metadata - metadata['start date'] = start_date - metadata['station identification number'] = int(date_line[:3]) - metadata['version of data'] = int(date_line.split()[-1]) - for line in fbuf: - if line[2:6] == '0004': # stop once LR0004 has been reached - break - elif line == '': - raise ValueError('Mandatory record LR0004 not found.') - metadata['date when station description changed'] = fbuf.readline().strip() - metadata['surface type'] = int(fbuf.readline(3)) - metadata['topography type'] = int(fbuf.readline()) - metadata['address'] = fbuf.readline().strip() - metadata['telephone no. of station'] = fbuf.readline(20).strip() - metadata['FAX no. of station'] = fbuf.readline().strip() - metadata['TCP/IP no. of station'] = fbuf.readline(15).strip() - metadata['e-mail address of station'] = fbuf.readline().strip() - metadata['latitude_bsrn'] = float(fbuf.readline(8)) # BSRN convention - metadata['latitude'] = metadata['latitude_bsrn'] - 90 # ISO 19115 - metadata['longitude_bsrn'] = float(fbuf.readline(8)) # BSRN convention - metadata['longitude'] = metadata['longitude_bsrn'] - 180 # ISO 19115 - metadata['altitude'] = int(fbuf.readline(5)) - metadata['identification of "SYNOP" station'] = fbuf.readline().strip() - metadata['date when horizon changed'] = fbuf.readline().strip() - # Pass last section of LR0004 containing the horizon elevation data - horizon = [] # list for raw horizon elevation data - while True: - line = fbuf.readline() - if ('*' in line) | (line == ''): - break - else: - horizon += [int(i) for i in line.split()] - horizon = pd.Series(horizon[1::2], horizon[::2], name='horizon_elevation', - dtype=int).drop(-1, errors='ignore').sort_index() - horizon.index.name = 'azimuth' - metadata['horizon'] = horizon - - # Read file and store the starting line number and number of lines for - # each logical record (LR) - fbuf.seek(0) # reset buffer to start of file - lr_startrow = {} # Dictionary of starting line number for each LR - lr_nrows = {} # Dictionary of end line number for each LR - for num, line in enumerate(fbuf): - if line.startswith('*'): # Find start of all logical records - if len(lr_startrow) >= 1: - lr_nrows[lr] = num - lr_startrow[lr] - 1 # noqa: F821 - lr = line[2:6] # string of 4 digit LR number - lr_startrow[lr] = num - lr_nrows[lr] = num - lr_startrow[lr] - - for lr in logical_records: - if lr not in ['0100', '0300', '0500']: - raise ValueError(f"Logical record {lr} not in " - "['0100', '0300','0500'].") - dfs = [] # Initialize empty list for dataframe - - # Parse LR0100 - basic measurements including GHI, DNI, DHI and temperature - if ('0100' in lr_startrow.keys()) & ('0100' in logical_records): - fbuf.seek(0) # reset buffer to start of file - LR_0100 = pd.read_fwf(fbuf, skiprows=lr_startrow['0100'] + 1, - nrows=lr_nrows['0100'], header=None, - colspecs=BSRN_LR0100_COL_SPECS, - na_values=[-999.0, -99.9]) - # Create multi-index and unstack, resulting in 1 col for each variable - LR_0100 = LR_0100.set_index([LR_0100.index // 2, LR_0100.index % 2]) - LR_0100 = LR_0100.unstack(level=1).swaplevel(i=0, j=1, axis='columns') - # Sort columns to match original order and assign column names - LR_0100 = LR_0100.reindex(sorted(LR_0100.columns), axis='columns') - LR_0100.columns = BSRN_LR0100_COLUMNS - # Set datetime index - LR_0100.index = (start_date+pd.to_timedelta(LR_0100['day']-1, unit='D') - + pd.to_timedelta(LR_0100['minute'], unit='minutes')) - # Drop empty, minute, and day columns - LR_0100 = LR_0100.drop(columns=['empty', 'day', 'minute']) - dfs.append(LR_0100) - - # Parse LR0300 - other time series data, including upward and net radiation - if ('0300' in lr_startrow.keys()) & ('0300' in logical_records): - fbuf.seek(0) # reset buffer to start of file - LR_0300 = pd.read_fwf(fbuf, skiprows=lr_startrow['0300']+1, - nrows=lr_nrows['0300'], header=None, - na_values=[-999.0, -99.9], - colspecs=BSRN_LR0300_COL_SPECS, - names=BSRN_LR0300_COLUMNS) - LR_0300.index = (start_date+pd.to_timedelta(LR_0300['day']-1, unit='D') - + pd.to_timedelta(LR_0300['minute'], unit='minutes')) - LR_0300 = LR_0300.drop(columns=['day', 'minute']).astype(float) - dfs.append(LR_0300) - - # Parse LR0500 - UV measurements - if ('0500' in lr_startrow.keys()) & ('0500' in logical_records): - fbuf.seek(0) # reset buffer to start of file - LR_0500 = pd.read_fwf(fbuf, skiprows=lr_startrow['0500']+1, - nrows=lr_nrows['0500'], na_values=[-99.9], - header=None, colspecs=BSRN_LR0500_COL_SPECS) - # Create multi-index and unstack, resulting in 1 col for each variable - LR_0500 = LR_0500.set_index([LR_0500.index // 2, LR_0500.index % 2]) - LR_0500 = LR_0500.unstack(level=1).swaplevel(i=0, j=1, axis='columns') - # Sort columns to match original order and assign column names - LR_0500 = LR_0500.reindex(sorted(LR_0500.columns), axis='columns') - LR_0500.columns = BSRN_LR0500_COLUMNS - LR_0500.index = (start_date+pd.to_timedelta(LR_0500['day']-1, unit='D') - + pd.to_timedelta(LR_0500['minute'], unit='minutes')) - LR_0500 = LR_0500.drop(columns=['empty', 'day', 'minute']) - dfs.append(LR_0500) - - if len(dfs): - data = pd.concat(dfs, axis='columns', sort=False) - else: - data = _empty_dataframe_from_logical_records(logical_records) - metadata = {} - return data, metadata - - def read_bsrn(filename, logical_records=('0100',)): """ Read a BSRN station-to-archive file into a DataFrame. @@ -459,5 +312,135 @@ def read_bsrn(filename, logical_records=('0100',)): else: open_func, mode = _file_context_manager, 'r' with open_func(filename, mode) as f: - content = _parse_bsrn(f, logical_records) - return content + # Parse metadata + f.readline() # first line should be *U0001, so read it and discard + date_line = f.readline() # second line contains important metadata + start_date = pd.Timestamp(year=int(date_line[7:11]), + month=int(date_line[3:6]), day=1, + tz='UTC') # BSRN timestamps are UTC + + metadata = {} # Initilize dictionary containing metadata + metadata['start date'] = start_date + metadata['station identification number'] = int(date_line[:3]) + metadata['version of data'] = int(date_line.split()[-1]) + for line in f: + if line[2:6] == '0004': # stop once LR0004 has been reached + break + elif line == '': + raise ValueError('Mandatory record LR0004 not found.') + metadata['date when station description changed'] = ( + f.readline().strip()) + metadata['surface type'] = int(f.readline(3)) + metadata['topography type'] = int(f.readline()) + metadata['address'] = f.readline().strip() + metadata['telephone no. of station'] = f.readline(20).strip() + metadata['FAX no. of station'] = f.readline().strip() + metadata['TCP/IP no. of station'] = f.readline(15).strip() + metadata['e-mail address of station'] = f.readline().strip() + metadata['latitude_bsrn'] = float(f.readline(8)) # BSRN convention + metadata['latitude'] = metadata['latitude_bsrn'] - 90 # ISO 19115 + metadata['longitude_bsrn'] = float(f.readline(8)) # BSRN convention + metadata['longitude'] = metadata['longitude_bsrn'] - 180 # ISO 19115 + metadata['altitude'] = int(f.readline(5)) + metadata['identification of "SYNOP" station'] = f.readline().strip() + metadata['date when horizon changed'] = f.readline().strip() + # Pass last section of LR0004 containing the horizon elevation data + horizon = [] # list for raw horizon elevation data + while True: + line = f.readline() + if ('*' in line) | (line == ''): + break + else: + horizon += [int(i) for i in line.split()] + horizon = pd.Series(horizon[1::2], horizon[::2], + name='horizon_elevation', dtype=int).drop( + -1, errors='ignore').sort_index() + horizon.index.name = 'azimuth' + metadata['horizon'] = horizon + + # Read file and store the starting line number and number of lines for + # each logical record (LR) + f.seek(0) # reset buffer to start of file + lr_startrow = {} # Dictionary of starting line number for each LR + lr_nrows = {} # Dictionary of end line number for each LR + for num, line in enumerate(f): + if line.startswith('*'): # Find start of all logical records + if len(lr_startrow) >= 1: + lr_nrows[lr] = num - lr_startrow[lr] - 1 # noqa: F821 + lr = line[2:6] # string of 4 digit LR number + lr_startrow[lr] = num + lr_nrows[lr] = num - lr_startrow[lr] + + for lr in logical_records: + if lr not in ['0100', '0300', '0500']: + raise ValueError(f"Logical record {lr} not in " + "['0100', '0300','0500'].") + dfs = [] # Initialize empty list for dataframe + + # Parse LR0100 - basic measurements including GHI, DNI, DHI and + # temperature + if ('0100' in lr_startrow.keys()) & ('0100' in logical_records): + f.seek(0) # reset buffer to start of file + LR_0100 = pd.read_fwf(f, skiprows=lr_startrow['0100'] + 1, + nrows=lr_nrows['0100'], header=None, + colspecs=BSRN_LR0100_COL_SPECS, + na_values=[-999.0, -99.9]) + # Create multi-index and unstack, resulting in 1 col for each + # variable + LR_0100 = LR_0100.set_index( + [LR_0100.index // 2, LR_0100.index % 2]) + LR_0100 = LR_0100.unstack(level=1).swaplevel( + i=0, j=1, axis='columns') + # Sort columns to match original order and assign column names + LR_0100 = LR_0100.reindex(sorted(LR_0100.columns), axis='columns') + LR_0100.columns = BSRN_LR0100_COLUMNS + # Set datetime index + LR_0100.index = ( + start_date + pd.to_timedelta(LR_0100['day']-1, unit='D') + + pd.to_timedelta(LR_0100['minute'], unit='minutes')) + # Drop empty, minute, and day columns + LR_0100 = LR_0100.drop(columns=['empty', 'day', 'minute']) + dfs.append(LR_0100) + + # Parse LR0300 - other time series data, including upward and net + # radiation + if ('0300' in lr_startrow.keys()) & ('0300' in logical_records): + f.seek(0) # reset buffer to start of file + LR_0300 = pd.read_fwf(f, skiprows=lr_startrow['0300']+1, + nrows=lr_nrows['0300'], header=None, + na_values=[-999.0, -99.9], + colspecs=BSRN_LR0300_COL_SPECS, + names=BSRN_LR0300_COLUMNS) + LR_0300.index = ( + start_date + pd.to_timedelta(LR_0300['day']-1, unit='D') + + pd.to_timedelta(LR_0300['minute'], unit='minutes')) + LR_0300 = LR_0300.drop(columns=['day', 'minute']).astype(float) + dfs.append(LR_0300) + + # Parse LR0500 - UV measurements + if ('0500' in lr_startrow.keys()) & ('0500' in logical_records): + f.seek(0) # reset buffer to start of file + LR_0500 = pd.read_fwf(f, skiprows=lr_startrow['0500']+1, + nrows=lr_nrows['0500'], na_values=[-99.9], + header=None, colspecs=BSRN_LR0500_COL_SPECS) + # Create multi-index and unstack, resulting in 1 col for each + # variable + LR_0500 = LR_0500.set_index( + [LR_0500.index // 2, LR_0500.index % 2]) + LR_0500 = LR_0500.unstack(level=1).swaplevel( + i=0, j=1, axis='columns') + # Sort columns to match original order and assign column names + LR_0500 = LR_0500.reindex(sorted(LR_0500.columns), axis='columns') + LR_0500.columns = BSRN_LR0500_COLUMNS + LR_0500.index = ( + start_date + pd.to_timedelta(LR_0500['day']-1, unit='D') + + pd.to_timedelta(LR_0500['minute'], unit='minutes')) + LR_0500 = LR_0500.drop(columns=['empty', 'day', 'minute']) + dfs.append(LR_0500) + + if len(dfs): + data = pd.concat(dfs, axis='columns', sort=False) + else: + data = _empty_dataframe_from_logical_records(logical_records) + metadata = {} + return data, metadata diff --git a/pvlib/iotools/epw.py b/pvlib/iotools/epw.py index 01c87c180a..53a0b43dc6 100644 --- a/pvlib/iotools/epw.py +++ b/pvlib/iotools/epw.py @@ -224,87 +224,51 @@ def read_epw(filename, coerce_year=None): 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 ' 'Safari/537.36')}) response = urlopen(request) - with io.StringIO(response.read().decode(errors='ignore')) as csvdata: - data, meta = _parse_epw(csvdata, coerce_year) - - else: - # Assume it's a buffer or accessible via the file system - with _file_context_manager(filename, 'r') as csvdata: - data, meta = _parse_epw(csvdata, coerce_year) - - return data, meta - - -def _parse_epw(csvdata, coerce_year=None): - """ - Given a file-like buffer with data in Energy Plus Weather (EPW) format, - parse the data into a dataframe. - - Parameters - ---------- - csvdata : file-like buffer - a file-like buffer containing data in the EPW format - - coerce_year : int, optional - If supplied, the year of the data will be set to this value. This can - be a useful feature because EPW data is composed of data from - different years. - Warning: EPW files always have 365*24 = 8760 data rows; - be careful with the use of leap years. - - Returns - ------- - data : DataFrame - A pandas dataframe with the columns described in the table - below. For more detailed descriptions of each component, please - consult the EnergyPlus Auxiliary Programs documentation - available at: https://energyplus.readthedocs.io/en/latest/. - - metadata : dict - The site metadata available in the file. - - See Also - -------- - pvlib.iotools.read_epw - """ - # Read line with metadata - firstline = csvdata.readline() - - head = ['loc', 'city', 'state-prov', 'country', 'data_type', 'WMO_code', - 'latitude', 'longitude', 'TZ', 'altitude'] - meta = dict(zip(head, firstline.rstrip('\n').split(","))) - - meta['altitude'] = float(meta['altitude']) - meta['latitude'] = float(meta['latitude']) - meta['longitude'] = float(meta['longitude']) - meta['TZ'] = float(meta['TZ']) - - colnames = ['year', 'month', 'day', 'hour', 'minute', 'data_source_unct', - 'temp_air', 'temp_dew', 'relative_humidity', - 'atmospheric_pressure', 'etr', 'etrn', 'ghi_infrared', 'ghi', - 'dni', 'dhi', 'global_hor_illum', 'direct_normal_illum', - 'diffuse_horizontal_illum', 'zenith_luminance', - 'wind_direction', 'wind_speed', 'total_sky_cover', - 'opaque_sky_cover', 'visibility', 'ceiling_height', - 'present_weather_observation', 'present_weather_codes', - 'precipitable_water', 'aerosol_optical_depth', 'snow_depth', - 'days_since_last_snowfall', 'albedo', - 'liquid_precipitation_depth', 'liquid_precipitation_quantity'] - - # We only have to skip 6 rows instead of 7 because we have already used - # the realine call above. - data = pd.read_csv(csvdata, skiprows=6, header=0, names=colnames) - - # Change to single year if requested - if coerce_year is not None: - data["year"] = coerce_year - - # create index that supplies correct date and time zone information - dts = data[['month', 'day']].astype(str).apply(lambda x: x.str.zfill(2)) - hrs = (data['hour'] - 1).astype(str).str.zfill(2) - dtscat = data['year'].astype(str) + dts['month'] + dts['day'] + hrs - idx = pd.to_datetime(dtscat, format='%Y%m%d%H') - idx = idx.dt.tz_localize(int(meta['TZ'] * 3600)) - data.index = idx - - return data, meta + filename = io.StringIO(response.read().decode(errors='ignore')) + + # Assume it's a buffer or accessible via the file system + with _file_context_manager(filename, 'r') as csvdata: + # Read line with metadata + firstline = csvdata.readline() + + head = ['loc', 'city', 'state-prov', 'country', 'data_type', + 'WMO_code', 'latitude', 'longitude', 'TZ', 'altitude'] + meta = dict(zip(head, firstline.rstrip('\n').split(","))) + + meta['altitude'] = float(meta['altitude']) + meta['latitude'] = float(meta['latitude']) + meta['longitude'] = float(meta['longitude']) + meta['TZ'] = float(meta['TZ']) + + colnames = ['year', 'month', 'day', 'hour', 'minute', + 'data_source_unct', 'temp_air', 'temp_dew', + 'relative_humidity', 'atmospheric_pressure', 'etr', + 'etrn', 'ghi_infrared', 'ghi', 'dni', 'dhi', + 'global_hor_illum', 'direct_normal_illum', + 'diffuse_horizontal_illum', 'zenith_luminance', + 'wind_direction', 'wind_speed', 'total_sky_cover', + 'opaque_sky_cover', 'visibility', 'ceiling_height', + 'present_weather_observation', 'present_weather_codes', + 'precipitable_water', 'aerosol_optical_depth', + 'snow_depth', 'days_since_last_snowfall', 'albedo', + 'liquid_precipitation_depth', + 'liquid_precipitation_quantity'] + + # We only have to skip 6 rows instead of 7 because we have already used + # the realine call above. + data = pd.read_csv(csvdata, skiprows=6, header=0, names=colnames) + + # Change to single year if requested + if coerce_year is not None: + data["year"] = coerce_year + + # create index that supplies correct date and time zone information + dts = data[['month', 'day']].astype(str).apply( + lambda x: x.str.zfill(2)) + hrs = (data['hour'] - 1).astype(str).str.zfill(2) + dtscat = data['year'].astype(str) + dts['month'] + dts['day'] + hrs + idx = pd.to_datetime(dtscat, format='%Y%m%d%H') + idx = idx.dt.tz_localize(int(meta['TZ'] * 3600)) + data.index = idx + + return data, meta From 9e902883f748d14800bbf1d3691c81f21e842fe0 Mon Sep 17 00:00:00 2001 From: DresdenGman Date: Tue, 3 Nov 2026 15:26:42 +0000 Subject: [PATCH 2/2] Add mocked FTP and HTTP tests for get_bsrn and read_epw --- tests/iotools/test_bsrn.py | 67 ++++++++++++++++++++++++++++++++++++++ tests/iotools/test_epw.py | 43 ++++++++++++++++++++++++ 2 files changed, 110 insertions(+) diff --git a/tests/iotools/test_bsrn.py b/tests/iotools/test_bsrn.py index 6b5f356836..2d2b17c922 100644 --- a/tests/iotools/test_bsrn.py +++ b/tests/iotools/test_bsrn.py @@ -4,6 +4,7 @@ import pandas as pd import pytest +import io import os import tempfile from pvlib.iotools import read_bsrn, get_bsrn @@ -148,3 +149,69 @@ def test_get_bsrn_no_files(bsrn_credentials): station='tam', username=username, password=password) + + +def _mock_bsrn_ftp(mocker, payload): + """Patch ftplib.FTP so that get_bsrn 'downloads' the given bytes.""" + def retrbinary(cmd, callback): + callback(payload) + return '226 Transfer complete.' + + mock_ftp_class = mocker.patch('pvlib.iotools.bsrn.ftplib.FTP') + mock_ftp = mock_ftp_class.return_value.__enter__.return_value + mock_ftp.retrbinary.side_effect = retrbinary + return mock_ftp + + +def test_get_bsrn_mocked_ftp(expected_index, mocker): + # Serve a local test file through a mocked FTP connection; the + # result must match read_bsrn on the same file exactly. + with open(TESTS_DATA_DIR / 'bsrn-pay0616.dat.gz', 'rb') as f: + payload = f.read() + mock_ftp = _mock_bsrn_ftp(mocker, payload) + + data, metadata = get_bsrn(station='pay', + start=pd.Timestamp(2016, 6, 1), + end=pd.Timestamp(2016, 6, 29), + username='user', password='pass') + + mock_ftp.cwd.assert_called_once_with('/pay') + assert mock_ftp.retrbinary.call_count == 1 + assert (mock_ftp.retrbinary.call_args.args[0] + == 'RETR pay0616.dat.gz') + expected_data, expected_metadata = read_bsrn( + TESTS_DATA_DIR / 'bsrn-pay0616.dat.gz') + pd.testing.assert_frame_equal(data, expected_data) + assert metadata.keys() == expected_metadata.keys() + for key, expected_value in expected_metadata.items(): + if isinstance(expected_value, pd.Series): + pd.testing.assert_series_equal(metadata[key], expected_value) + else: + assert metadata[key] == expected_value + assert_index_equal(expected_index, data.index) + + +def test_get_bsrn_mocked_ftp_logical_records(mocker): + # Non-default logical records must be forwarded to read_bsrn, which + # get_bsrn calls with the downloaded gzip TextIOWrapper. + with open(TESTS_DATA_DIR / 'bsrn-pay0616.dat.gz', 'rb') as f: + payload = f.read() + logical_records = ('0100', '0300', '0500') + expected_data, _ = read_bsrn(TESTS_DATA_DIR / 'bsrn-pay0616.dat.gz', + logical_records=logical_records) + _mock_bsrn_ftp(mocker, payload) + spy = mocker.patch('pvlib.iotools.bsrn.read_bsrn', wraps=read_bsrn) + + data, metadata = get_bsrn(station='pay', + start=pd.Timestamp(2016, 6, 1), + end=pd.Timestamp(2016, 6, 29), + username='user', password='pass', + logical_records=logical_records) + + spy.assert_called_once() + buffer_arg, lr_arg = spy.call_args.args + assert isinstance(buffer_arg, io.TextIOWrapper) + assert lr_arg == logical_records + # LR0300 content proves the records were actually parsed + assert 'lwu' in data.columns + pd.testing.assert_frame_equal(data, expected_data) diff --git a/tests/iotools/test_epw.py b/tests/iotools/test_epw.py index 9f1dd48ff4..dc9d471bef 100644 --- a/tests/iotools/test_epw.py +++ b/tests/iotools/test_epw.py @@ -1,3 +1,4 @@ +import pandas as pd import pytest from pvlib.iotools import epw @@ -32,3 +33,45 @@ def test_read_epw_coerce_year(): coerce_year = 1987 data, _ = epw.read_epw(epw_testfile, coerce_year=coerce_year) assert (data.index.year == 1987).all() + + +def _mock_epw_urlopen(mocker, payload): + """Patch pvlib.iotools.epw.urlopen to serve the given bytes.""" + mock_response = mocker.MagicMock() + mock_response.read.return_value = payload + return mocker.patch('pvlib.iotools.epw.urlopen', + return_value=mock_response) + + +def test_read_epw_http(mocker): + # Mock the download; the HTTP branch must parse exactly like the + # local file entry point. + with open(epw_testfile, 'rb') as f: + payload = f.read() + mock_urlopen = _mock_epw_urlopen(mocker, payload) + + url = 'https://example.com/NLD_Amsterdam062400_IWEC.epw' + df, meta = epw.read_epw(url) + + request = mock_urlopen.call_args.args[0] + assert request.full_url == url + expected_df, expected_meta = epw.read_epw(epw_testfile) + pd.testing.assert_frame_equal(df, expected_df) + assert meta == expected_meta + + +def test_read_epw_http_coerce_year(mocker): + # coerce_year must be honored on the HTTP branch as well. + with open(epw_testfile, 'rb') as f: + payload = f.read() + _mock_epw_urlopen(mocker, payload) + + url = 'https://example.com/NLD_Amsterdam062400_IWEC.epw' + df, meta = epw.read_epw(url, coerce_year=2015) + + assert (df['year'] == 2015).all() + assert (df.index.year == 2015).all() + expected_df, expected_meta = epw.read_epw(epw_testfile, + coerce_year=2015) + pd.testing.assert_frame_equal(df, expected_df) + assert meta == expected_meta