Brian Blaylock
July 20, 2021
RAP Data#
There are different products available on the cloud.
[1]:
from herbie.archive import Herbie
from toolbox.cartopy_tools import common_features, pc
from paint.standard2 import cm_tmp
import pandas as pd
import matplotlib.pyplot as plt
import cartopy.crs as ccrs
[3]:
H = Herbie("2021-11-19", model="rap", product="awp130pgrb")
🏋🏻♂️ Found 2021-Nov-19 00:00 UTC F00 [RAP] [product=awp130pgrb] GRIB2 file from aws and index file from aws.
[4]:
x = H.xarray("TMP:2 m above")
👨🏻🏭 Created directory: [/p/cwfs/blaylock/data/rap/20211119]
📇 Download subset: [RAP] model [awp130pgrb] product run at 2021-Nov-19 00:00 UTC F00
cURL from https://noaa-rap-pds.s3.amazonaws.com/rap.20211119/rap.t00z.awp130pgrbf00.grib2
1: GRIB_message=203 TMP:2 m above ground:anl
[5]:
ax = common_features(crs=x.herbie.crs, figsize=[8, 8]).ax
p = ax.pcolormesh(
x.longitude, x.latitude, x.t2m, transform=pc, **cm_tmp(units="K").cmap_kwargs
)
plt.colorbar(
p, ax=ax, orientation="horizontal", pad=0.05, **cm_tmp(units="K").cbar_kwargs
)
ax.set_title(x.t2m.GRIB_name, loc="right")
ax.set_title(f"{x.model.upper()}: {H.product_description}", loc="left")
/p/home/blaylock/anaconda3/envs/herbie/lib/python3.9/site-packages/metpy/xarray.py:349: UserWarning: More than one time coordinate present for variable "gribfile_projection".
warnings.warn('More than one ' + axis + ' coordinate present for variable'
/p/home/blaylock/anaconda3/envs/herbie/lib/python3.9/site-packages/cartopy/mpl/geoaxes.py:1702: UserWarning: The input coordinates to pcolormesh are interpreted as cell centers, but are not monotonically increasing or decreasing. This may lead to incorrectly calculated cell edges, in which case, please supply explicit cell edges to pcolormesh.
X, Y, C, shading = self._pcolorargs('pcolormesh', *args,
[5]:
Text(0.0, 1.0, 'RAP: CONUS Pressure levels; 13-km resolution')

RAP record on NCEI#
[9]:
H = Herbie("2019-11-23", model="rap_historical", product="analysis")
🏋🏻♂️ Found 2019-Nov-23 00:00 UTC F00 [RAP_HISTORICAL] [product=analysis] GRIB2 file from rap_130 and index file from rap_130.
[11]:
x = H.xarray("TMP:2 m above")
👨🏻🏭 Created directory: [/p/cwfs/blaylock/data/rap_historical/20191123]
📇 Download subset: [RAP_HISTORICAL] model [analysis] product run at 2019-Nov-23 00:00 UTC F00
cURL from https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/201911/20191123/rap_130_20191123_0000_000.grb2
1: GRIB_message=203 TMP:2 m above ground:anl
[12]:
ax = common_features(crs=x.herbie.crs, figsize=[8, 8]).ax
p = ax.pcolormesh(
x.longitude, x.latitude, x.t2m, transform=pc, **cm_tmp(units="K").cmap_kwargs
)
plt.colorbar(
p, ax=ax, orientation="horizontal", pad=0.05, **cm_tmp(units="K").cbar_kwargs
)
ax.set_title(x.t2m.GRIB_name, loc="right")
ax.set_title(f"{x.model.upper()}: {H.product_description}", loc="left")
/p/home/blaylock/anaconda3/envs/herbie/lib/python3.9/site-packages/metpy/xarray.py:349: UserWarning: More than one time coordinate present for variable "gribfile_projection".
warnings.warn('More than one ' + axis + ' coordinate present for variable'
/p/home/blaylock/anaconda3/envs/herbie/lib/python3.9/site-packages/cartopy/mpl/geoaxes.py:1702: UserWarning: The input coordinates to pcolormesh are interpreted as cell centers, but are not monotonically increasing or decreasing. This may lead to incorrectly calculated cell edges, in which case, please supply explicit cell edges to pcolormesh.
X, Y, C, shading = self._pcolorargs('pcolormesh', *args,
[12]:
Text(0.0, 1.0, 'RAP_HISTORICAL: January 2005 through May 2020')

[15]:
H._check_grib(H.grib)
[15]:
True
[16]:
H.grib
[16]:
'https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/201911/20191123/rap_130_20191123_0000_000.grb2'
[19]:
H = Herbie("2019-11-19", model="rap_historical", product="analysis")
assert H.grib is not None
⚠ Herbie didn't find any inventory files that exists from ['.inv', '.grb2.inv', 'grb.inv']
⚠ Herbie didn't find any inventory files that exists from ['.inv', '.grb2.inv', 'grb.inv']
⚠ Herbie didn't find any inventory files that exists from ['.inv', '.grb2.inv', 'grb.inv']
⚠ Herbie didn't find any inventory files that exists from ['.inv', '.grb2.inv', 'grb.inv']
⚠ Herbie didn't find any inventory files that exists from ['.inv', '.grb2.inv', 'grb.inv']
💔 Did not find a GRIB2 or Index File for 2019-Nov-19 00:00 UTC F00 RAP_HISTORICAL
---------------------------------------------------------------------------
AssertionError Traceback (most recent call last)
/p/work1/tmp/blaylock/ipykernel_68742/2558624910.py in <module>
1 H = Herbie('2019-11-19', model='rap_historical', product='analysis')
----> 2 assert H.grib is not None
AssertionError:
[21]:
H = Herbie("2005-01-01", model="rap_historical", product="analysis")
assert H.grib is not None
⚠ Herbie didn't find any inventory files that exists from ['.inv', '.grb2.inv', 'grb.inv']
⚠ Herbie didn't find any inventory files that exists from ['.inv', '.grb2.inv', 'grb.inv']
🏋🏻♂️ Found 2005-Jan-01 00:00 UTC F00 [RAP_HISTORICAL] [product=analysis] GRIB2 file from ruc_252 and index file from ruc_252.
[26]:
H = Herbie("2020-03-15", model="rap_ncei", product="rap-130-13km")
H.SOURCES
🏋🏻♂️ Found 2020-Mar-15 00:00 UTC F00 [RAP_NCEI] [product=rap-130-13km] GRIB2 file from ncei_13km_analysis and index file from ncei_13km_analysis.
[26]:
{'ncei_13km_analysis': 'https://www.ncei.noaa.gov/data/rapid-refresh/access/rap-130-13km/analysis/202003/20200315/rap_130_20200315_0000_000.grb2',
'ncei_13km_forecast': 'https://www.ncei.noaa.gov/data/rapid-refresh/access/rap-130-13km/forecast/202003/20200315/rap_130_20200315_0000_000.grb2',
'ncei_20km_analysis': 'https://www.ncei.noaa.gov/data/rapid-refresh/access/rap-130-13km/analysis/202003/20200315/rap_252_20200315_0000_000.grb2',
'ncei_20km_forecast': 'https://www.ncei.noaa.gov/data/rapid-refresh/access/rap-130-13km/forecast/202003/20200315/rap_252_20200315_0000_000.grb2'}
[30]:
import pandas as pd
H = Herbie(pd.to_datetime("today").floor("1D"), model="rap")
/p/home/blaylock/BB_python/Herbie/herbie/archive.py:217: UserWarning: `product` not specified. Will use ["awp130pgrb"].
warnings.warn(f'`product` not specified. Will use ["{self.product}"].')
🏋🏻♂️ Found 2021-Dec-01 00:00 UTC F00 [RAP] [product=awp130pgrb] GRIB2 file from aws and index file from aws.
[29]:
pd.to_datetime("today").floor("1D")
[29]:
Timestamp('2021-12-01 00:00:00')
[31]:
from datetime import datetime
now = datetime.now()
today = datetime(now.year, now.month, now.day)
today_str = today.strftime("%Y-%m-%d %H:%M")
[32]:
H = Herbie(today_str, model="rap")
/p/home/blaylock/BB_python/Herbie/herbie/archive.py:217: UserWarning: `product` not specified. Will use ["awp130pgrb"].
warnings.warn(f'`product` not specified. Will use ["{self.product}"].')
🏋🏻♂️ Found 2021-Dec-01 00:00 UTC F00 [RAP] [product=awp130pgrb] GRIB2 file from aws and index file from aws.
[33]:
H.download()
👨🏻🏭 Created directory: [/p/cwfs/blaylock/data/rap/20211201]
✅ Success! Downloaded RAP from aws
src: https://noaa-rap-pds.s3.amazonaws.com/rap.20211201/rap.t00z.awp130pgrbf00.grib2
dst: /p/cwfs/blaylock/data/rap/20211201/rap.t00z.awp130pgrbf00.grib2
[34]:
H.xarray("TMP:2 m")
📇 Download subset: [RAP] model [awp130pgrb] product run at 2021-Dec-01 00:00 UTC F00
cURL from https://noaa-rap-pds.s3.amazonaws.com/rap.20211201/rap.t00z.awp130pgrbf00.grib2
1: GRIB_message=203 TMP:2 m above ground:anl
[34]:
<xarray.Dataset> Dimensions: (y: 337, x: 451) Coordinates: time datetime64[ns] 2021-12-01 step timedelta64[ns] 00:00:00 heightAboveGround float64 2.0 latitude (y, x) float64 16.28 16.31 16.34 ... 55.54 55.51 55.48 longitude (y, x) float64 233.9 234.0 234.1 ... 302.3 302.4 302.6 valid_time datetime64[ns] 2021-12-01 Dimensions without coordinates: y, x Data variables: t2m (y, x) float32 298.0 298.0 297.9 ... 273.7 273.6 273.6 gribfile_projection object None Attributes: GRIB_edition: 2 GRIB_centre: kwbc GRIB_centreDescription: US National Weather Service - NCEP GRIB_subCentre: 0 Conventions: CF-1.7 institution: US National Weather Service - NCEP model: rap product: awp130pgrb description: Rapid Refresh (RAP) from NOMADS and Big Data Pro... remote_grib: https://noaa-rap-pds.s3.amazonaws.com/rap.202112... local_grib: /p/cwfs/blaylock/data/rap/20211201/rap.t00z.awp1...
xarray.Dataset
- y: 337
- x: 451
- time()datetime64[ns]2021-12-01
- long_name :
- initial time of forecast
- standard_name :
- forecast_reference_time
array('2021-12-01T00:00:00.000000000', dtype='datetime64[ns]')
- step()timedelta64[ns]00:00:00
- long_name :
- time since forecast_reference_time
- standard_name :
- forecast_period
array(0, dtype='timedelta64[ns]')
- heightAboveGround()float642.0
- long_name :
- height above the surface
- units :
- m
- positive :
- up
- standard_name :
- height
array(2.)
- latitude(y, x)float6416.28 16.31 16.34 ... 55.51 55.48
- units :
- degrees_north
- standard_name :
- latitude
- long_name :
- latitude
array([[16.281 , 16.30836614, 16.33562845, ..., 17.38598091, 17.363161 , 17.34023363], [16.39828505, 16.42568412, 16.45297923, ..., 17.50458529, 17.48173835, 17.45878381], [16.51559943, 16.54303139, 16.57035924, ..., 17.62321703, 17.60034309, 17.57736144], ..., [53.97394127, 54.00790887, 54.04173934, ..., 55.33884573, 55.31079595, 55.2826082 ], [54.07324369, 54.10721488, 54.14104889, ..., 55.43826705, 55.41021535, 55.38202566], [54.17241813, 54.20639281, 54.24023027, ..., 55.53755643, 55.5095029 , 55.48131134]])
- longitude(y, x)float64233.9 234.0 234.1 ... 302.4 302.6
- units :
- degrees_east
- standard_name :
- longitude
- long_name :
- longitude
array([[233.862 , 233.98419514, 234.1064416 , ..., 290.71357615, 290.83782217, 290.96202426], [233.83341078, 233.95571014, 234.07806096, ..., 290.73745411, 290.86180986, 290.98612158], [233.80476994, 233.9271737 , 234.04962904, ..., 290.76137592, 290.8858416 , 291.01026312], ..., [220.2599064 , 220.42892476, 220.59808701, ..., 302.16983749, 302.34470507, 302.51944524], [220.20196465, 220.37116928, 220.54051831, ..., 302.21910048, 302.3941759 , 302.56912343], [220.14387816, 220.31326939, 220.48280555, ..., 302.26849092, 302.44377462, 302.61892995]])
- valid_time()datetime64[ns]2021-12-01
- standard_name :
- time
- long_name :
- time
array('2021-12-01T00:00:00.000000000', dtype='datetime64[ns]')
- t2m(y, x)float32298.0 298.0 297.9 ... 273.6 273.6
- GRIB_paramId :
- 167
- GRIB_dataType :
- fc
- GRIB_numberOfPoints :
- 151987
- GRIB_typeOfLevel :
- heightAboveGround
- GRIB_stepUnits :
- 1
- GRIB_stepType :
- instant
- GRIB_gridType :
- lambert
- GRIB_DxInMetres :
- 13545.0
- GRIB_DyInMetres :
- 13545.0
- GRIB_LaDInDegrees :
- 25.0
- GRIB_Latin1InDegrees :
- 25.0
- GRIB_Latin2InDegrees :
- 25.0
- GRIB_LoVInDegrees :
- 265.0
- GRIB_NV :
- 0
- GRIB_Nx :
- 451
- GRIB_Ny :
- 337
- GRIB_cfName :
- air_temperature
- GRIB_cfVarName :
- t2m
- GRIB_gridDefinitionDescription :
- Lambert Conformal can be secant or tangent, conical or bipolar
- GRIB_iScansNegatively :
- 0
- GRIB_jPointsAreConsecutive :
- 0
- GRIB_jScansPositively :
- 1
- GRIB_latitudeOfFirstGridPointInDegrees :
- 16.281
- GRIB_latitudeOfSouthernPoleInDegrees :
- 0.0
- GRIB_longitudeOfFirstGridPointInDegrees :
- 233.862
- GRIB_longitudeOfSouthernPoleInDegrees :
- 0.0
- GRIB_missingValue :
- 9999
- GRIB_name :
- 2 metre temperature
- GRIB_parameterName :
- Temperature
- GRIB_parameterUnits :
- K
- GRIB_shortName :
- 2t
- GRIB_stepRange :
- 0
- GRIB_units :
- K
- long_name :
- 2 metre temperature
- units :
- K
- standard_name :
- air_temperature
- grid_mapping :
- gribfile_projection
array([[298.00787, 298.00787, 297.94537, ..., 301.82037, 301.82037, 301.82037], [297.82037, 297.88287, 297.82037, ..., 301.75787, 301.82037, 301.82037], [297.75787, 297.75787, 297.75787, ..., 301.63287, 301.69537, 301.69537], ..., [279.88287, 279.94537, 280.00787, ..., 273.88287, 273.88287, 273.82037], [279.82037, 279.88287, 279.94537, ..., 273.82037, 273.75787, 273.75787], [279.82037, 279.88287, 279.94537, ..., 273.69537, 273.63287, 273.63287]], dtype=float32)
- gribfile_projection()objectNone
- crs_wkt :
- PROJCRS["unknown",BASEGEOGCRS["unknown",DATUM["unknown",ELLIPSOID["unknown",6371229,0,LENGTHUNIT["metre",1,ID["EPSG",9001]]]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8901]]],CONVERSION["unknown",METHOD["Lambert Conic Conformal (2SP)",ID["EPSG",9802]],PARAMETER["Latitude of false origin",25,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8821]],PARAMETER["Longitude of false origin",265,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8822]],PARAMETER["Latitude of 1st standard parallel",25,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8823]],PARAMETER["Latitude of 2nd standard parallel",25,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8824]],PARAMETER["Easting at false origin",0,LENGTHUNIT["metre",1],ID["EPSG",8826]],PARAMETER["Northing at false origin",0,LENGTHUNIT["metre",1],ID["EPSG",8827]]],CS[Cartesian,2],AXIS["(E)",east,ORDER[1],LENGTHUNIT["metre",1,ID["EPSG",9001]]],AXIS["(N)",north,ORDER[2],LENGTHUNIT["metre",1,ID["EPSG",9001]]]]
- semi_major_axis :
- 6371229.0
- semi_minor_axis :
- 6371229.0
- inverse_flattening :
- 0.0
- reference_ellipsoid_name :
- unknown
- longitude_of_prime_meridian :
- 0.0
- prime_meridian_name :
- Greenwich
- geographic_crs_name :
- unknown
- horizontal_datum_name :
- unknown
- projected_crs_name :
- unknown
- grid_mapping_name :
- lambert_conformal_conic
- standard_parallel :
- (25.0, 25.0)
- latitude_of_projection_origin :
- 25.0
- longitude_of_central_meridian :
- 265.0
- false_easting :
- 0.0
- false_northing :
- 0.0
- long_name :
- RAP model grid projection
array(None, dtype=object)
- GRIB_edition :
- 2
- GRIB_centre :
- kwbc
- GRIB_centreDescription :
- US National Weather Service - NCEP
- GRIB_subCentre :
- 0
- Conventions :
- CF-1.7
- institution :
- US National Weather Service - NCEP
- model :
- rap
- product :
- awp130pgrb
- description :
- Rapid Refresh (RAP) from NOMADS and Big Data Program
- remote_grib :
- https://noaa-rap-pds.s3.amazonaws.com/rap.20211201/rap.t00z.awp130pgrbf00.grib2
- local_grib :
- /p/cwfs/blaylock/data/rap/20211201/rap.t00z.awp130pgrbf00.grib2.subset_a165fbd61c277745f187eaac7182d9c05d0d1171
List files that are found in Historical NCEI#
[4]:
DATES = pd.date_range("2005-01-01", "today", freq="AS")
len(DATES), DATES
[4]:
(17,
DatetimeIndex(['2005-01-01', '2006-01-01', '2007-01-01', '2008-01-01',
'2009-01-01', '2010-01-01', '2011-01-01', '2012-01-01',
'2013-01-01', '2014-01-01', '2015-01-01', '2016-01-01',
'2017-01-01', '2018-01-01', '2019-01-01', '2020-01-01',
'2021-01-01'],
dtype='datetime64[ns]', freq='AS-JAN'))
[6]:
for D in DATES:
H = Herbie(D, model="rap_historical", verbose=False, product="analysis")
print(f"{D:%Y-%m-%d %H:%M} ->FOUND-> {H.grib}")
2005-01-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/200501/20050101/ruc2_252_20050101_0000_000.grb
2006-01-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/200601/20060101/ruc2_252_20060101_0000_000.grb
2007-01-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/200701/20070101/ruc2anl_252_20070101_0000_000.grb
2008-01-01 00:00 ->FOUND-> None
2009-01-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/200901/20090101/ruc2anl_252_20090101_0000_000.grb
2010-01-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/201001/20100101/ruc2anl_252_20100101_0000_000.grb
2011-01-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/201101/20110101/ruc2anl_252_20110101_0000_000.grb
2012-01-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/201201/20120101/ruc2anl_252_20120101_0000_000.grb
2013-01-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/201301/20130101/rap_130_20130101_0000_000.grb2
2014-01-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/201401/20140101/rap_130_20140101_0000_000.grb2
2015-01-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/201501/20150101/rap_130_20150101_0000_000.grb2
2016-01-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/201601/20160101/rap_130_20160101_0000_000.grb2
2017-01-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/201701/20170101/rap_130_20170101_0000_000.grb2
2018-01-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/201801/20180101/rap_130_20180101_0000_000.grb2
2019-01-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/201901/20190101/rap_252_20190101_0000_000.grb2
2020-01-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/analysis/202001/20200101/rap_130_20200101_0000_000.grb2
2021-01-01 00:00 ->FOUND-> None
[12]:
DATES = pd.date_range("2003-01-01", "today", freq="AS-FEB")
print(len(DATES), DATES)
19 DatetimeIndex(['2003-02-01', '2004-02-01', '2005-02-01', '2006-02-01',
'2007-02-01', '2008-02-01', '2009-02-01', '2010-02-01',
'2011-02-01', '2012-02-01', '2013-02-01', '2014-02-01',
'2015-02-01', '2016-02-01', '2017-02-01', '2018-02-01',
'2019-02-01', '2020-02-01', '2021-02-01'],
dtype='datetime64[ns]', freq='AS-FEB')
[13]:
for D in DATES:
H = Herbie(D, model="rap_historical", verbose=False, product="forecast")
print(f"{D:%Y-%m-%d %H:%M} ->FOUND-> {H.grib}")
2003-02-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/forecast/200302/20030201/ruc2_236_20030201_0000_000.grb
2004-02-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/forecast/200402/20040201/ruc2_236_20040201_0000_000.grb
2005-02-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/forecast/200502/20050201/ruc2_236_20050201_0000_000.grb
2006-02-01 00:00 ->FOUND-> None
2007-02-01 00:00 ->FOUND-> None
2008-02-01 00:00 ->FOUND-> None
2009-02-01 00:00 ->FOUND-> None
2010-02-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/forecast/201002/20100201/ruc2_252_20100201_0000_000.grb
2011-02-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/forecast/201102/20110201/ruc2_252_20110201_0000_000.grb
2012-02-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/forecast/201202/20120201/ruc2_252_20120201_0000_000.grb
2013-02-01 00:00 ->FOUND-> None
2014-02-01 00:00 ->FOUND-> None
2015-02-01 00:00 ->FOUND-> None
2016-02-01 00:00 ->FOUND-> None
2017-02-01 00:00 ->FOUND-> None
2018-02-01 00:00 ->FOUND-> None
2019-02-01 00:00 ->FOUND-> None
2020-02-01 00:00 ->FOUND-> https://www.ncei.noaa.gov/data/rapid-refresh/access/historical/forecast/202002/20200201/rap_252_20200201_0000_000.grb2
2021-02-01 00:00 ->FOUND-> None
[ ]: