Getting the NASA bio-Optical Marine Algorithm Dataset (NOMAD) into a Pandas DataFrame

import requests
import pandas as pd
import re
import numpy as np
import pickle
from IPython.core.display import display, HTML

display(HTML("<style>.container {width:90% !important;}</style>"))

NOMADV2url='https://seabass.gsfc.nasa.gov/wiki/NOMAD/nomad_seabass_v2.a_2008200.txt'

def GetNomad(url=NOMADV2url):
    """Download and return data as text"""
    resp = requests.get(NOMADV2url)
    content = resp.text.splitlines()    
    resp.close()
    return content

def ParseTextFile(textFile, topickle=False, convert2DateTime=False, **kwargs):
    """
    * topickle: pickle resulting DataFrame if True
    * convert2DateTime: join date/time columns and convert entries to datetime objects
    * kwargs:
        pkl_fname: pickle file name to save DataFrame by, if topickle=True
    """
    # Pre-compute some regex
    columns = re.compile('^/fields=(.+)') # to get field/column names
    units = re.compile('^/units=(.+)') # to get units -- optional
    endHeader = re.compile('^/end_header') # to know when to start storing data
    # Set some milestones
    noFields = True
    getData = False
    # loop through the text data
    for line in textFile:
        if noFields:
            fieldStr = columns.findall(line)
            if len(fieldStr)>0:
                noFields = False
                fieldList = fieldStr[0].split(',')
                dataDict = dict.fromkeys(fieldList)
                continue # nothing left to do with this line, keep looping
        if not getData:
            if endHeader.match(line):
                # end of header reached, start acquiring data
                getData = True 
        else:
            dataList = line.split(',')
            for field,datum in zip(fieldList, dataList):
                if not dataDict[field]:
                    dataDict[field] = []
                dataDict[field].append(datum)
    df = pd.DataFrame(dataDict, columns=fieldList)
    if convert2DateTime:
        datetimelabels=['year', 'month', 'day', 'hour', 'minute', 'second']
        df['Datetime']= pd.to_datetime(df[datetimelabels],
                                       format='%Y-%m-%dT%H:%M:%S')
        df.drop(datetimelabels, axis=1, inplace=True)
    if topickle:
        fname=kwargs.pop('pkl_fname', 'dfNomad2.pkl')
        df.to_pickle(fname)
    return df

df = ParseTextFile(GetNomad(), topickle=True, convert2DateTime=True,
                  pkl_fname='./bayesianChl_DATA/dfNomadRaw.pkl')

df.head()

bandregex = re.compile('es([0-9]+)')
bands = bandregex.findall(''.join(df.columns))
print(bands)

swfBands = ['411','443','489','510','555','670']
dfSwf = pd.DataFrame(columns=['rrs%s' % b for b in swfBands])
for b in swfBands:
    dfSwf.loc[:,'rrs%s'%b] = df.loc[:,'lw%s' % b].astype('f8') / df.loc[:,'es%s' % b].astype('f8')

dfSwf.head()

dfSwf['id'] = df.id.astype('i4') # in case I need to relate this data to the original
dfSwf['datetime'] = df.Datetime
dfSwf['hplc_chl'] = df.chl_a.astype('f8')
dfSwf['fluo_chl'] = df.chl.astype('f8')
dfSwf['lat'] = df.lat.astype('f8')
dfSwf['lon'] = df.lon.astype('f8')
dfSwf['depth'] = df.etopo2.astype('f8')
dfSwf['sst'] = df.oisst.astype('f8')
for band in swfBands:
    addprods=['a','ad','ag','ap','bb']
    for prod in addprods:
        dfSwf['%s%s' % (prod,band)] = df['%s%s' % (prod, band)].astype('f8')
dfSwf.replace(-999,np.nan, inplace=True)

print(dfSwf.columns)

dfSwf.to_pickle('./bayesianChl_DATA/dfNomadSWF.pkl')

rrsCols = [col for col in dfSwf.columns if 'rrs' in col]
iwantcols=rrsCols + ['id', 'depth','hplc_chl','sst','lat','lon']
dfSwfHu = dfSwf[iwantcols].copy()
del dfSwf, df

dfSwfHu.info()

dfSwfHu.dropna(inplace=True)

dfSwfHu.describe()

dfSwfHu=dfSwfHu.loc[((dfSwfHu.depth>30) &\
                     (dfSwfHu.lat>=-60) & (dfSwfHu.lat<=60)),:]

dfSwfHu.describe()

dfSwfHu = dfSwfHu.loc[((dfSwfHu.rrs411<1.0) & (dfSwfHu.rrs510<1.0)&\
                               (dfSwfHu.rrs555<1.0) & (dfSwfHu.rrs670<1.0)),:]

dfSwfHu.describe()

dfSwfHu.to_pickle('/accounts/ekarakoy/DATA/NOMAD/dfSwfHuOcxCI_2012.pkl')