Brian Blaylock
May 6, 2022
Historical NOGAPS Data#
From NCEI for historical analyses from 1997 to 2008 at 0.5 and 1.0 degree grids.
[1]:
import cartopy.crs as ccrs
import matplotlib.pyplot as plt
from paint.standard2 import cm_tmp
from toolbox.cartopy_tools import common_features, pc
from herbie import Herbie
[9]:
H = Herbie("2008-07-28 12:00", model="nogaps", product="058_240")
✅ Found ┊ model=nogaps ┊ product=058_240 ┊ 2008-Jul-28 12:00 UTC F00 ┊ GRIB2 @ ncei ┊ IDX @ ncei
[10]:
# TODO: Herbie cant read NOGAPS inventory files
H.read_idx("TMP")
---------------------------------------------------------------------------
ParserError Traceback (most recent call last)
c:\Users\blayl_depgywe\BB_python\Herbie\docs\user_guide\notebooks\data_nogaps.ipynb Cell 4' in <cell line: 1>()
----> <a href='vscode-notebook-cell:/c%3A/Users/blayl_depgywe/BB_python/Herbie/docs/user_guide/notebooks/data_nogaps.ipynb#ch0000009?line=0'>1</a> H.read_idx("TMP")
File ~\BB_python\Herbie\herbie\archive.py:661, in Herbie.read_idx(self, searchString)
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=639'>640</a> def read_idx(self, searchString=None):
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=640'>641</a> """
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=641'>642</a> Inspect the GRIB2 file contents by reading the index file.
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=642'>643</a>
(...)
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=658'>659</a> A Pandas DataFrame of the index file.
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=659'>660</a> """
--> <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=660'>661</a> df = self.index_as_dataframe
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=662'>663</a> # Filter DataFrame by searchString
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=663'>664</a> if searchString not in [None, ":"]:
File ~\miniconda3\envs\herbie\lib\functools.py:993, in cached_property.__get__(self, instance, owner)
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/functools.py?line=990'>991</a> val = cache.get(self.attrname, _NOT_FOUND)
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/functools.py?line=991'>992</a> if val is _NOT_FOUND:
--> <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/functools.py?line=992'>993</a> val = self.func(instance)
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/functools.py?line=993'>994</a> try:
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/functools.py?line=994'>995</a> cache[self.attrname] = val
File ~\BB_python\Herbie\herbie\archive.py:519, in Herbie.index_as_dataframe(self)
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=509'>510</a> assert self.idx is not None, f"No index file found for {self.grib}."
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=511'>512</a> if self.IDX_STYLE == "wgrib2":
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=512'>513</a> # Sometimes idx end in ':', other times it doesn't (in some Pando files).
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=513'>514</a> # https://pando-rgw01.chpc.utah.edu/hrrr/sfc/20180101/hrrr.t00z.wrfsfcf00.grib2.idx
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=514'>515</a> # https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20210101/conus/hrrr.t00z.wrfsfcf00.grib2.idx
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=515'>516</a> # Sometimes idx has more than the standard messages
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=516'>517</a> # https://noaa-nbm-grib2-pds.s3.amazonaws.com/blend.20210711/13/core/blend.t13z.core.f001.co.grib2.idx
--> <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=518'>519</a> df = pd.read_csv(
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=519'>520</a> self.idx,
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=520'>521</a> sep=":",
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=521'>522</a> names=[
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=522'>523</a> "grib_message",
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=523'>524</a> "start_byte",
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=524'>525</a> "reference_time",
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=525'>526</a> "variable",
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=526'>527</a> "level",
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=527'>528</a> "forecast_time",
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=528'>529</a> "?",
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=529'>530</a> "??",
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=530'>531</a> "???",
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=531'>532</a> ],
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=532'>533</a> )
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=534'>535</a> # Format the DataFrame
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=535'>536</a> df["reference_time"] = pd.to_datetime(
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=536'>537</a> df.reference_time, format="d=%Y%m%d%H"
<a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=537'>538</a> )
File ~\miniconda3\envs\herbie\lib\site-packages\pandas\util\_decorators.py:311, in deprecate_nonkeyword_arguments.<locals>.decorate.<locals>.wrapper(*args, **kwargs)
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/util/_decorators.py?line=304'>305</a> if len(args) > num_allow_args:
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/util/_decorators.py?line=305'>306</a> warnings.warn(
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/util/_decorators.py?line=306'>307</a> msg.format(arguments=arguments),
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/util/_decorators.py?line=307'>308</a> FutureWarning,
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/util/_decorators.py?line=308'>309</a> stacklevel=stacklevel,
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/util/_decorators.py?line=309'>310</a> )
--> <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/util/_decorators.py?line=310'>311</a> return func(*args, **kwargs)
File ~\miniconda3\envs\herbie\lib\site-packages\pandas\io\parsers\readers.py:680, in read_csv(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=664'>665</a> kwds_defaults = _refine_defaults_read(
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=665'>666</a> dialect,
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=666'>667</a> delimiter,
(...)
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=675'>676</a> defaults={"delimiter": ","},
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=676'>677</a> )
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=677'>678</a> kwds.update(kwds_defaults)
--> <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=679'>680</a> return _read(filepath_or_buffer, kwds)
File ~\miniconda3\envs\herbie\lib\site-packages\pandas\io\parsers\readers.py:581, in _read(filepath_or_buffer, kwds)
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=577'>578</a> return parser
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=579'>580</a> with parser:
--> <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=580'>581</a> return parser.read(nrows)
File ~\miniconda3\envs\herbie\lib\site-packages\pandas\io\parsers\readers.py:1254, in TextFileReader.read(self, nrows)
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=1251'>1252</a> nrows = validate_integer("nrows", nrows)
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=1252'>1253</a> try:
-> <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=1253'>1254</a> index, columns, col_dict = self._engine.read(nrows)
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=1254'>1255</a> except Exception:
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=1255'>1256</a> self.close()
File ~\miniconda3\envs\herbie\lib\site-packages\pandas\io\parsers\c_parser_wrapper.py:225, in CParserWrapper.read(self, nrows)
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/c_parser_wrapper.py?line=222'>223</a> try:
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/c_parser_wrapper.py?line=223'>224</a> if self.low_memory:
--> <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/c_parser_wrapper.py?line=224'>225</a> chunks = self._reader.read_low_memory(nrows)
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/c_parser_wrapper.py?line=225'>226</a> # destructive to chunks
<a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/c_parser_wrapper.py?line=226'>227</a> data = _concatenate_chunks(chunks)
File ~\miniconda3\envs\herbie\lib\site-packages\pandas\_libs\parsers.pyx:805, in pandas._libs.parsers.TextReader.read_low_memory()
File ~\miniconda3\envs\herbie\lib\site-packages\pandas\_libs\parsers.pyx:861, in pandas._libs.parsers.TextReader._read_rows()
File ~\miniconda3\envs\herbie\lib\site-packages\pandas\_libs\parsers.pyx:847, in pandas._libs.parsers.TextReader._tokenize_rows()
File ~\miniconda3\envs\herbie\lib\site-packages\pandas\_libs\parsers.pyx:1960, in pandas._libs.parsers.raise_parser_error()
ParserError: Error tokenizing data. C error: EOF inside string starting at row 74
[23]:
H = Herbie("2008-07-28 12:00", model="nogaps", product="008_240")
✅ Found ┊ model=nogaps ┊ product=008_240 ┊ 2008-Jul-28 12:00 UTC F00 ┊ GRIB2 @ ncei ┊ IDX @ ncei
[25]:
H.read_idx("TMP")
[25]:
grib_message | start_byte | end_byte | range | reference_time | valid_time | variable | level | forecast_time | ? | ?? | ??? | search_this | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 1 | 21 | 228297 | 21-228297 | 2008-07-28 12:00:00 | 2008-07-28 12:00:00 | TMP | 0 mb | kpds=11,100,0 | anl | winds are N/S | Temp. [K]\n2:89724:D=2008072812:HGT:0 mb:kpds=... | :TMP:0 mb:kpds=11,100,0:anl:winds are N/S:Temp... |
2 | 5 | 407705 | 644127 | 407705-644127 | 2008-07-28 12:00:00 | 2008-07-28 12:00:00 | TMP | 1 mb | kpds=11,100,1 | anl | winds are N/S | Temp. [K]\n6:505554:D=2008072812:HGT:1 mb:kpds... | :TMP:1 mb:kpds=11,100,1:anl:winds are N/S:Temp... |
4 | 9 | 823535 | 1068101 | 823535-1068101 | 2008-07-28 12:00:00 | 2008-07-28 12:00:00 | TMP | 2 mb | kpds=11,100,2 | anl | winds are N/S | Temp. [K]\n10:929528:D=2008072812:HGT:2 mb:kpd... | :TMP:2 mb:kpds=11,100,2:anl:winds are N/S:Temp... |
6 | 13 | 1231217 | 1467639 | 1231217-1467639 | 2008-07-28 12:00:00 | 2008-07-28 12:00:00 | TMP | 5 mb | kpds=11,100,5 | anl | winds are N/S | Temp. [K]\n14:1337210:D=2008072812:HGT:5 mb:kp... | :TMP:5 mb:kpds=11,100,5:anl:winds are N/S:Temp... |
[26]:
ds = H.xarray("TMP:5 mb")
C:\Users\blayl_depgywe\BB_python\Herbie\herbie\archive.py:983: UserWarning: sorry, on windows I couldn't remove the file.
warnings.warn("sorry, on windows I couldn't remove the file.")
[27]:
ds
[27]:
<xarray.Dataset> Dimensions: (latitude: 181, longitude: 360) Coordinates: time datetime64[ns] 2008-07-28T12:00:00 step timedelta64[ns] 00:00:00 isobaricInhPa float64 5.0 * latitude (latitude) float64 -90.0 -89.0 -88.0 ... 88.0 89.0 90.0 * longitude (longitude) float64 0.0 1.0 2.0 ... 357.0 358.0 359.0 valid_time datetime64[ns] 2008-07-28T12:00:00 Data variables: t (latitude, longitude) float32 210.5 210.5 ... 252.4 gh (latitude, longitude) float32 3.048e+04 ... 3.679e+04 gribfile_projection object None Attributes: GRIB_edition: 1 GRIB_centre: fnmo GRIB_centreDescription: US Navy - Fleet Numerical Oceanography Center GRIB_subCentre: 0 Conventions: CF-1.7 institution: US Navy - Fleet Numerical Oceanography Center model: nogaps product: 008_240 description: Navy Operational Global Atmospheric Prediction S... remote_grib: https://www.ncei.noaa.gov/data/navy-operational-... local_grib: C:\Users\blayl_depgywe\data\nogaps\20080728\subs... searchString: TMP:5 mb
[35]:
ax = common_features(crs=ds.herbie.crs, figsize=[8, 8]).ax
p = ax.pcolormesh(
ds.longitude, ds.latitude, ds.t, transform=pc, **cm_tmp(units="K").cmap_kwargs
)
plt.colorbar(
p, ax=ax, orientation="horizontal", pad=0.05, **cm_tmp(units="K").cbar_kwargs
)
ax.set_title(f"{ds.t.GRIB_name} {ds.isobaricInhPa.item()} hPa", loc="right")
ax.set_title(f"{H.model.upper()}: {H.product_description}", loc="left")
[35]:
Text(0.0, 1.0, 'NOGAPS: set1')

[ ]:
[ ]: