Brian Blaylock
May 6, 2022

Historical NOGAPS Data#

From NCEI for historical analyses from 1997 to 2008 at 0.5 and 1.0 degree grids.

[1]:
import cartopy.crs as ccrs
import matplotlib.pyplot as plt
from paint.standard2 import cm_tmp
from toolbox.cartopy_tools import common_features, pc

from herbie import Herbie
[9]:
H = Herbie("2008-07-28 12:00", model="nogaps", product="058_240")
✅ Found ┊ model=nogaps ┊ product=058_240 ┊ 2008-Jul-28 12:00 UTC F00GRIB2 @ nceiIDX @ ncei
[10]:
# TODO: Herbie cant read NOGAPS inventory files

H.read_idx("TMP")
---------------------------------------------------------------------------
ParserError                               Traceback (most recent call last)
c:\Users\blayl_depgywe\BB_python\Herbie\docs\user_guide\notebooks\data_nogaps.ipynb Cell 4' in <cell line: 1>()
----> <a href='vscode-notebook-cell:/c%3A/Users/blayl_depgywe/BB_python/Herbie/docs/user_guide/notebooks/data_nogaps.ipynb#ch0000009?line=0'>1</a> H.read_idx("TMP")

File ~\BB_python\Herbie\herbie\archive.py:661, in Herbie.read_idx(self, searchString)
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=639'>640</a> def read_idx(self, searchString=None):
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=640'>641</a>     """
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=641'>642</a>     Inspect the GRIB2 file contents by reading the index file.
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=642'>643</a>
   (...)
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=658'>659</a>     A Pandas DataFrame of the index file.
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=659'>660</a>     """
--> <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=660'>661</a>     df = self.index_as_dataframe
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=662'>663</a>     # Filter DataFrame by searchString
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=663'>664</a>     if searchString not in [None, ":"]:

File ~\miniconda3\envs\herbie\lib\functools.py:993, in cached_property.__get__(self, instance, owner)
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/functools.py?line=990'>991</a> val = cache.get(self.attrname, _NOT_FOUND)
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/functools.py?line=991'>992</a> if val is _NOT_FOUND:
--> <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/functools.py?line=992'>993</a>     val = self.func(instance)
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/functools.py?line=993'>994</a>     try:
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/functools.py?line=994'>995</a>         cache[self.attrname] = val

File ~\BB_python\Herbie\herbie\archive.py:519, in Herbie.index_as_dataframe(self)
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=509'>510</a> assert self.idx is not None, f"No index file found for {self.grib}."
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=511'>512</a> if self.IDX_STYLE == "wgrib2":
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=512'>513</a>     # Sometimes idx end in ':', other times it doesn't (in some Pando files).
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=513'>514</a>     # https://pando-rgw01.chpc.utah.edu/hrrr/sfc/20180101/hrrr.t00z.wrfsfcf00.grib2.idx
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=514'>515</a>     # https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20210101/conus/hrrr.t00z.wrfsfcf00.grib2.idx
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=515'>516</a>     # Sometimes idx has more than the standard messages
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=516'>517</a>     # https://noaa-nbm-grib2-pds.s3.amazonaws.com/blend.20210711/13/core/blend.t13z.core.f001.co.grib2.idx
--> <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=518'>519</a>     df = pd.read_csv(
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=519'>520</a>         self.idx,
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=520'>521</a>         sep=":",
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=521'>522</a>         names=[
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=522'>523</a>             "grib_message",
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=523'>524</a>             "start_byte",
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=524'>525</a>             "reference_time",
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=525'>526</a>             "variable",
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=526'>527</a>             "level",
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=527'>528</a>             "forecast_time",
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=528'>529</a>             "?",
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=529'>530</a>             "??",
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=530'>531</a>             "???",
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=531'>532</a>         ],
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=532'>533</a>     )
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=534'>535</a>     # Format the DataFrame
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=535'>536</a>     df["reference_time"] = pd.to_datetime(
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=536'>537</a>         df.reference_time, format="d=%Y%m%d%H"
    <a href='file:///c%3A/Users/blayl_depgywe/BB_python/Herbie/herbie/archive.py?line=537'>538</a>     )

File ~\miniconda3\envs\herbie\lib\site-packages\pandas\util\_decorators.py:311, in deprecate_nonkeyword_arguments.<locals>.decorate.<locals>.wrapper(*args, **kwargs)
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/util/_decorators.py?line=304'>305</a> if len(args) > num_allow_args:
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/util/_decorators.py?line=305'>306</a>     warnings.warn(
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/util/_decorators.py?line=306'>307</a>         msg.format(arguments=arguments),
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/util/_decorators.py?line=307'>308</a>         FutureWarning,
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/util/_decorators.py?line=308'>309</a>         stacklevel=stacklevel,
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/util/_decorators.py?line=309'>310</a>     )
--> <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/util/_decorators.py?line=310'>311</a> return func(*args, **kwargs)

File ~\miniconda3\envs\herbie\lib\site-packages\pandas\io\parsers\readers.py:680, in read_csv(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=664'>665</a> kwds_defaults = _refine_defaults_read(
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=665'>666</a>     dialect,
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=666'>667</a>     delimiter,
   (...)
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=675'>676</a>     defaults={"delimiter": ","},
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=676'>677</a> )
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=677'>678</a> kwds.update(kwds_defaults)
--> <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=679'>680</a> return _read(filepath_or_buffer, kwds)

File ~\miniconda3\envs\herbie\lib\site-packages\pandas\io\parsers\readers.py:581, in _read(filepath_or_buffer, kwds)
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=577'>578</a>     return parser
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=579'>580</a> with parser:
--> <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=580'>581</a>     return parser.read(nrows)

File ~\miniconda3\envs\herbie\lib\site-packages\pandas\io\parsers\readers.py:1254, in TextFileReader.read(self, nrows)
   <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=1251'>1252</a> nrows = validate_integer("nrows", nrows)
   <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=1252'>1253</a> try:
-> <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=1253'>1254</a>     index, columns, col_dict = self._engine.read(nrows)
   <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=1254'>1255</a> except Exception:
   <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/readers.py?line=1255'>1256</a>     self.close()

File ~\miniconda3\envs\herbie\lib\site-packages\pandas\io\parsers\c_parser_wrapper.py:225, in CParserWrapper.read(self, nrows)
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/c_parser_wrapper.py?line=222'>223</a> try:
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/c_parser_wrapper.py?line=223'>224</a>     if self.low_memory:
--> <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/c_parser_wrapper.py?line=224'>225</a>         chunks = self._reader.read_low_memory(nrows)
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/c_parser_wrapper.py?line=225'>226</a>         # destructive to chunks
    <a href='file:///c%3A/Users/blayl_depgywe/miniconda3/envs/herbie/lib/site-packages/pandas/io/parsers/c_parser_wrapper.py?line=226'>227</a>         data = _concatenate_chunks(chunks)

File ~\miniconda3\envs\herbie\lib\site-packages\pandas\_libs\parsers.pyx:805, in pandas._libs.parsers.TextReader.read_low_memory()

File ~\miniconda3\envs\herbie\lib\site-packages\pandas\_libs\parsers.pyx:861, in pandas._libs.parsers.TextReader._read_rows()

File ~\miniconda3\envs\herbie\lib\site-packages\pandas\_libs\parsers.pyx:847, in pandas._libs.parsers.TextReader._tokenize_rows()

File ~\miniconda3\envs\herbie\lib\site-packages\pandas\_libs\parsers.pyx:1960, in pandas._libs.parsers.raise_parser_error()

ParserError: Error tokenizing data. C error: EOF inside string starting at row 74
[23]:
H = Herbie("2008-07-28 12:00", model="nogaps", product="008_240")
✅ Found ┊ model=nogaps ┊ product=008_240 ┊ 2008-Jul-28 12:00 UTC F00GRIB2 @ nceiIDX @ ncei
[25]:
H.read_idx("TMP")
[25]:
grib_message start_byte end_byte range reference_time valid_time variable level forecast_time ? ?? ??? search_this
0 1 21 228297 21-228297 2008-07-28 12:00:00 2008-07-28 12:00:00 TMP 0 mb kpds=11,100,0 anl winds are N/S Temp. [K]\n2:89724:D=2008072812:HGT:0 mb:kpds=... :TMP:0 mb:kpds=11,100,0:anl:winds are N/S:Temp...
2 5 407705 644127 407705-644127 2008-07-28 12:00:00 2008-07-28 12:00:00 TMP 1 mb kpds=11,100,1 anl winds are N/S Temp. [K]\n6:505554:D=2008072812:HGT:1 mb:kpds... :TMP:1 mb:kpds=11,100,1:anl:winds are N/S:Temp...
4 9 823535 1068101 823535-1068101 2008-07-28 12:00:00 2008-07-28 12:00:00 TMP 2 mb kpds=11,100,2 anl winds are N/S Temp. [K]\n10:929528:D=2008072812:HGT:2 mb:kpd... :TMP:2 mb:kpds=11,100,2:anl:winds are N/S:Temp...
6 13 1231217 1467639 1231217-1467639 2008-07-28 12:00:00 2008-07-28 12:00:00 TMP 5 mb kpds=11,100,5 anl winds are N/S Temp. [K]\n14:1337210:D=2008072812:HGT:5 mb:kp... :TMP:5 mb:kpds=11,100,5:anl:winds are N/S:Temp...
[26]:
ds = H.xarray("TMP:5 mb")
C:\Users\blayl_depgywe\BB_python\Herbie\herbie\archive.py:983: UserWarning: sorry, on windows I couldn't remove the file.
  warnings.warn("sorry, on windows I couldn't remove the file.")
[27]:
ds
[27]:
<xarray.Dataset>
Dimensions:              (latitude: 181, longitude: 360)
Coordinates:
    time                 datetime64[ns] 2008-07-28T12:00:00
    step                 timedelta64[ns] 00:00:00
    isobaricInhPa        float64 5.0
  * latitude             (latitude) float64 -90.0 -89.0 -88.0 ... 88.0 89.0 90.0
  * longitude            (longitude) float64 0.0 1.0 2.0 ... 357.0 358.0 359.0
    valid_time           datetime64[ns] 2008-07-28T12:00:00
Data variables:
    t                    (latitude, longitude) float32 210.5 210.5 ... 252.4
    gh                   (latitude, longitude) float32 3.048e+04 ... 3.679e+04
    gribfile_projection  object None
Attributes:
    GRIB_edition:            1
    GRIB_centre:             fnmo
    GRIB_centreDescription:  US Navy - Fleet Numerical Oceanography Center
    GRIB_subCentre:          0
    Conventions:             CF-1.7
    institution:             US Navy - Fleet Numerical Oceanography Center
    model:                   nogaps
    product:                 008_240
    description:             Navy Operational Global Atmospheric Prediction S...
    remote_grib:             https://www.ncei.noaa.gov/data/navy-operational-...
    local_grib:              C:\Users\blayl_depgywe\data\nogaps\20080728\subs...
    searchString:            TMP:5 mb
[35]:
ax = common_features(crs=ds.herbie.crs, figsize=[8, 8]).ax
p = ax.pcolormesh(
    ds.longitude, ds.latitude, ds.t, transform=pc, **cm_tmp(units="K").cmap_kwargs
)
plt.colorbar(
    p, ax=ax, orientation="horizontal", pad=0.05, **cm_tmp(units="K").cbar_kwargs
)

ax.set_title(f"{ds.t.GRIB_name} {ds.isobaricInhPa.item()} hPa", loc="right")
ax.set_title(f"{H.model.upper()}: {H.product_description}", loc="left")
[35]:
Text(0.0, 1.0, 'NOGAPS: set1')
../../_images/user_guide_notebooks_data_nogaps_8_1.png
[ ]:

[ ]: