bump min pandas version to 0.15.0, update tests and docs, add map_variables option

Author: lboeman

.travis.yml

@@ -69,7 +69,7 @@ install:
         pip uninstall numpy --yes;
         pip uninstall pandas --yes;
         pip install --no-cache-dir numpy==1.10.1;
-        pip install --no-cache-dir pandas==0.14.0;
+        pip install --no-cache-dir pandas==0.15.0;
       fi
     - conda list
     - echo $PATH

docs/sphinx/source/whatsnew/v0.6.1.rst

@@ -9,6 +9,8 @@ release.
 **Python 2.7 support will end on June 1, 2019**. Releases made after this
 date will require Python 3. (:issue:`501`)
 
+**Minimum pandas requirment bumped 0.14.0=>0.15.0**
+
 
 API Changes
 ~~~~~~~~~~~

pvlib/iotools/surfrad.py

@@ -23,14 +23,36 @@
     'temp', 'temp_flag', 'rh', 'rh_flag', 'windspd', 'windspd_flag',
     'winddir', 'winddir_flag', 'pressure', 'pressure_flag']
 
-
-def read_surfrad(filename):
+# Dictionary mapping surfrad variables to pvlib names
+VARIABLE_MAP = {
+    'zen': 'solar_zenith',
+    'dw_solar': 'ghi',
+    'dw_solar_flag': 'ghi_flag',
+    'direct_n': 'dni',
+    'direct_n_flag': 'dni_flag',
+    'diffuse': 'dhi',
+    'diffuse_flag': 'dhi_flag',
+    'temp': 'temp_air',
+    'temp_flag': 'temp_air_flag',
+    'windspd': 'wind_speed',
+    'windspd_flag': 'wind_speed_flag',
+    'winddir': 'wind_direction',
+    'winddir_flag': 'wind_direction_flag',
+    'rh': 'relative_humidity',
+    'rh_flag': 'relative_humidity_flag'
+}
+
+
+def read_surfrad(filename, map_variables=True):
     """Read in a daily NOAA SURFRAD[1] file.
 
     Parameters
     ----------
     filename: str
         Filepath or url.
+    map_variables: bool
+        When true, renames columns of the Dataframe to pvlib variable names
+        where applicable.
 
     Returns
     -------
@@ -53,44 +75,47 @@ def read_surfrad(filename):
     longitude        Float   site longitude
     elevation        Int     site elevation
     surfrad_version  Int     surfrad version
+    tz               String  Timezone (UTC)
     ===============  ======  ===============
 
     Dataframe includes the following fields:
 
-    ==================   ======  ==========================================
-    SURFRAD data Field   Format  Description
-    ==================   ======  ==========================================
-    year                 int     year as 4 digit int
-    jday                 int     day of year 1-365(or 366)
-    month                int     month (1-12)
-    day                  int     day of month(1-31)
-    hour                 int     hour (0-23)
-    minute               int     minute (0-59)
-    dt                   float   decimal time i.e. 23.5 = 2330
-    zen                  float   solar zenith angle (deg)
+    =======================  ======  ===========================================
+    raw, mapped              Format  Description
+    =======================  ======  ===========================================
+    **Mapped field names are returned when the map_variables argument is True**
+    ----------------------------------------------------------------------------
+    year                     int     year as 4 digit int
+    jday                     int     day of year 1-365(or 366)
+    month                    int     month (1-12)
+    day                      int     day of month(1-31)
+    hour                     int     hour (0-23)
+    minute                   int     minute (0-59)
+    dt                       float   decimal time i.e. 23.5 = 2330
+    zen, solar_zenith        float   solar zenith angle (deg)
     **Fields below have associated qc flags labeled <field>_flag.**
-    -----------------------------------------------------------------------
-    dw_solar             float   downwelling global solar(W/m^2)
-    uw_solar             float   updownwelling global solar(W/m^2)
-    direct_n             float   direct normal solar (W/m^2)
-    diffuse              float   downwelling diffuse solar (W/m^2)
-    dw_ir                float   downwelling thermal infrared (W/m^2)
-    dw_casetemp          float   downwelling IR case temp (K)
-    dw_dometemp          float   downwelling IR dome temp (K)
-    uw_ir                float   upwelling thermal infrared (W/m^2)
-    uw_casetemp          float   upwelling IR case temp (K)
-    uw_dometemp          float   upwelling IR case temp (K)
-    uvb                  float   global uvb (miliWatts/m^2)
-    par                  float   photosynthetically active radiation(W/m^2)
-    netsolar             float   net solar (dw_solar - uw_solar) (W/m^2)
-    netir                float   net infrared (dw_ir - uw_ir) (W/m^2)
-    totalnet             float   net radiation (netsolar+netir) (W/m^2)
-    temp                 float   10-meter air temperature (?C)
-    rh                   float   relative humidity (%)
-    windspd              float   wind speed (m/s)
-    winddir              float   wind direction (deg, clockwise from north)
-    pressure             float   station pressure (mb)
-    ==================   ======  ==========================================
+    ----------------------------------------------------------------------------
+    dw_solar, ghi            float   downwelling global solar(W/m^2)
+    uw_solar                 float   updownwelling global solar(W/m^2)
+    direct_n, dni            float   direct normal solar (W/m^2)
+    diffuse, dhi             float   downwelling diffuse solar (W/m^2)
+    dw_ir                    float   downwelling thermal infrared (W/m^2)
+    dw_casetemp              float   downwelling IR case temp (K)
+    dw_dometemp              float   downwelling IR dome temp (K)
+    uw_ir                    float   upwelling thermal infrared (W/m^2)
+    uw_casetemp              float   upwelling IR case temp (K)
+    uw_dometemp              float   upwelling IR case temp (K)
+    uvb                      float   global uvb (miliWatts/m^2)
+    par                      float   photosynthetically active radiation(W/m^2)
+    netsolar                 float   net solar (dw_solar - uw_solar) (W/m^2)
+    netir                    float   net infrared (dw_ir - uw_ir) (W/m^2)
+    totalnet                 float   net radiation (netsolar+netir) (W/m^2)
+    temp, temp_air           float   10-meter air temperature (?C)
+    rh, relative_humidity    float   relative humidity (%)
+    windspd, wind_speed      float   wind speed (m/s)
+    winddir, wind_direction  float   wind direction (deg, clockwise from north)
+    pressure                 float   station pressure (mb)
+    =======================  ======  ===========================================
 
     See README files located in the station directories in the SURFRAD
     data archives[2] for details on SURFRAD daily data files.
@@ -116,10 +141,11 @@ def read_surfrad(filename):
     metadata_list = file_metadata.split()
     metadata = {}
     metadata['name'] = station.strip()
-    metadata['latitude'] = metadata_list[0]
-    metadata['longitude'] = metadata_list[1]
-    metadata['elevation'] = metadata_list[2]
-    metadata['surfrad_version'] = metadata_list[-1]
+    metadata['latitude'] = float(metadata_list[0])
+    metadata['longitude'] = float(metadata_list[1])
+    metadata['elevation'] = float(metadata_list[2])
+    metadata['surfrad_version'] = int(metadata_list[-1])
+    metadata['tz'] = 'UTC'
 
     data = pd.read_csv(file_buffer, delim_whitespace=True,
                        header=None, names=SURFRAD_COLUMNS)
@@ -129,6 +155,8 @@ def read_surfrad(filename):
     missing = data == -9999.9
     data = data.where(~missing, np.NaN)
 
+    if map_variables:
+        data.rename(columns=VARIABLE_MAP, inplace=True)
     return data, metadata
 
 
@@ -150,8 +178,7 @@ def format_index(data):
     jday = data.jday.apply(lambda x: '{:03d}'.format(x))
     hours = data.hour.apply(lambda x: '{:02d}'.format(x))
     minutes = data.minute.apply(lambda x: '{:02d}'.format(x))
-
     index = pd.to_datetime(year + jday + hours + minutes, format="%Y%j%H%M")
     data.index = index
-    data.tz_localize('UTC')
+    data = data.tz_localize('UTC')
     return data

pvlib/test/test_surfrad.py

@@ -13,36 +13,53 @@
                     'Alamosa_CO/2016/slv16001.dat')
 
 
-def test_read_surfrad():
-    surfrad.read_surfrad(testfile)
-
-
 @network
 def test_read_surfrad_network():
-    surfrad.read_surfrad(network_testfile)
+    # If this test begins failing, SURFRAD's data structure or data
+    # archive may have changed.
+    local_data, _ = surfrad.read_surfrad(testfile)
+    network_data, _ = surfrad.read_surfrad(network_testfile)
+    assert local_data.equals(network_data)
 
 
-def test_read_surfrad_columns_exist():
-    data, _ = surfrad.read_surfrad(testfile)
+def test_read_surfrad_columns_no_map():
+    data, _ = surfrad.read_surfrad(testfile, map_variables=False)
     assert 'zen' in data.columns
     assert 'temp' in data.columns
     assert 'par' in data.columns
     assert 'pressure' in data.columns
 
 
+def test_read_surfrad_columns_map():
+    data, _ = surfrad.read_surfrad(testfile)
+    assert 'solar_zenith' in data.columns
+    assert 'ghi' in data.columns
+    assert 'ghi_flag' in data.columns
+    assert 'dni' in data.columns
+    assert 'dni_flag' in data.columns
+    assert 'dhi' in data.columns
+    assert 'dhi_flag' in data.columns
+    assert 'wind_direction' in data.columns
+    assert 'wind_direction_flag' in data.columns
+    assert 'wind_speed' in data.columns
+    assert 'wind_speed_flag' in data.columns
+    assert 'temp_air' in data.columns
+    assert 'temp_air_flag' in data.columns
+
+
 def test_format_index():
     start = Timestamp('20160101 00:00')
-    expected_index = DatetimeIndex(start=start, periods=1440, freq='1min')
-    expected_index.tz_localize('UTC')
+    expected = DatetimeIndex(start=start, periods=1440, freq='1min', tz='UTC')
     actual, _ = surfrad.read_surfrad(testfile)
-    assert actual.index.equals(expected_index)
+    assert actual.index.equals(expected)
 
 
 def test_read_surfrad_metadata():
     expected = {'name': 'Alamosa',
-                'latitude': '37.70',
-                'longitude': '105.92',
-                'elevation': '2317',
-                'surfrad_version': '1'}
+                'latitude': 37.70,
+                'longitude': 105.92,
+                'elevation': 2317,
+                'surfrad_version': 1,
+                'tz': 'UTC'}
     _, metadata = surfrad.read_surfrad(testfile)
     assert metadata == expected

setup.py

@@ -38,7 +38,7 @@
 URL = 'https://github.com/pvlib/pvlib-python'
 
 INSTALL_REQUIRES = ['numpy >= 1.10.1',
-                    'pandas >= 0.14.0',
+                    'pandas >= 0.15.0',
                     'pytz',
                     'six',
                     ]