22Useful utilities for data munging.
33"""
44import warnings
5+
56import pandas as pd
67import requests
8+
79import dataretrieval
810from dataretrieval .codes import tz
911
12+
1013def to_str (listlike , delimiter = ',' ):
1114 """Translates list-like objects into strings.
1215
@@ -74,23 +77,25 @@ def format_datetime(df, date_field, time_field, tz_field):
7477 # create a datetime index from the columns in qwdata response
7578 df [tz_field ] = df [tz_field ].map (tz )
7679
77- df ['datetime' ] = pd .to_datetime (df [ date_field ] + ' ' +
78- df [time_field ] + ' ' +
79- df [ tz_field ] ,
80- format = 'ISO8601' ,
81- utc = True )
80+ df ['datetime' ] = pd .to_datetime (
81+ df [ date_field ] + ' ' + df [time_field ] + ' ' + df [ tz_field ],
82+ format = 'ISO8601' ,
83+ utc = True ,
84+ )
8285
8386 # if there are any incomplete dates, warn the user
84- if any ( pd . isna ( df ['datetime' ]) ):
85- count = sum ( pd . isna ( df ['datetime' ]) == True )
87+ if df ['datetime' ]. isna (). any ( ):
88+ count = df ['datetime' ]. isna (). sum ( )
8689 warnings .warn (
87- f'Warning: { count } +
88- 'consider setting datetime_index to False.' , UserWarning )
90+ f'Warning: { count }
91+ + 'consider setting datetime_index to False.' ,
92+ UserWarning ,
93+ )
8994
9095 return df
9196
9297
93- #This function may be deprecated once pandas.update support joins besides left.
98+ # This function may be deprecated once pandas.update support joins besides left.
9499def update_merge (left , right , na_only = False , on = None , ** kwargs ):
95100 """Performs a combination update and merge.
96101
@@ -113,30 +118,30 @@ def update_merge(left, right, na_only=False, on=None, **kwargs):
113118 add na_only parameter support
114119
115120 """
116- #df = left.merge(right, how='outer',
121+ # df = left.merge(right, how='outer',
117122 # left_index=True, right_index=True)
118123 df = left .merge (right , how = 'outer' , on = on , ** kwargs )
119124
120-
121125 # check for column overlap and resolve update
122126 for column in df .columns :
123- #if duplicated column, use the value from right
127+ # if duplicated column, use the value from right
124128 if column [- 2 :] == '_x' :
125- name = column [:- 2 ] # find column name
129+ name = column [:- 2 ] # find column name
126130
127131 if na_only :
128- df [name ] = df [name + '_x' ].fillna (df [name + '_y' ])
132+ df [name ] = df [name + '_x' ].fillna (df [name + '_y' ])
129133
130134 else :
131- df [name ] = df [name + '_x' ].update (df [name + '_y' ])
135+ df [name ] = df [name + '_x' ].update (df [name + '_y' ])
132136
133137 df .drop ([name + '_x' , name + '_y' ], axis = 1 , inplace = True )
134138
135139 return df
136140
141+
137142class BaseMetadata :
138143 """Base class for metadata.
139-
144+
140145 Attributes
141146 ----------
142147 url : str
@@ -145,9 +150,9 @@ class BaseMetadata:
145150 Response elapsed time
146151 header: requests.structures.CaseInsensitiveDict
147152 Response headers
148-
153+
149154 """
150-
155+
151156 def __init__ (self , response ) -> None :
152157 """Generates a standard set of metadata informed by the response.
153158
@@ -168,30 +173,29 @@ def __init__(self, response) -> None:
168173 self .query_time = response .elapsed
169174 self .header = response .headers
170175 self .comment = None
171-
176+
172177 # # not sure what statistic_info is
173178 # self.statistic_info = None
174-
179+
175180 # # disclaimer seems to be only part of importWaterML1
176181 # self.disclaimer = None
177-
182+
178183 # These properties are to be set by `nwis` or `wqp`-specific metadata classes.
179184 @property
180185 def site_info (self ):
181186 raise NotImplementedError (
182- " site_info must be implemented by utils.BaseMetadata children"
187+ ' site_info must be implemented by utils.BaseMetadata children'
183188 )
184-
189+
185190 @property
186191 def variable_info (self ):
187192 raise NotImplementedError (
188- " variable_info must be implemented by utils.BaseMetadata children"
193+ ' variable_info must be implemented by utils.BaseMetadata children'
189194 )
190195
191-
192196 def __repr__ (self ) -> str :
193- return f" { type (self ).__name__ } { self .url } "
194-
197+ return f' { type (self ).__name__ } { self .url } '
198+
195199
196200def query (url , payload , delimiter = ',' , ssl_check = True ):
197201 """Send a query.
@@ -219,37 +223,40 @@ def query(url, payload, delimiter=',', ssl_check=True):
219223
220224 for key , value in payload .items ():
221225 payload [key ] = to_str (value , delimiter )
222- #for index in range(len(payload)):
226+ # for index in range(len(payload)):
223227 # key, value = payload[index]
224228 # payload[index] = (key, to_str(value))
225229
226230 # define the user agent for the query
227- user_agent = {
228- 'user-agent' : f"python-dataretrieval/{ dataretrieval .__version__ } }
231+ user_agent = {'user-agent' : f'python-dataretrieval/{ dataretrieval .__version__ } }
229232
230- response = requests .get (url , params = payload ,
231- headers = user_agent , verify = ssl_check )
233+ response = requests .get (url , params = payload , headers = user_agent , verify = ssl_check )
232234
233235 if response .status_code == 400 :
234- raise ValueError ("Bad Request, check that your parameters are correct. URL: {}" .format (response .url ))
236+ raise ValueError (
237+ f'Bad Request, check that your parameters are correct. URL: { response .url }
238+ )
235239 elif response .status_code == 404 :
236240 raise ValueError (
237- "Page Not Found Error. May be the result of an empty query. " +
238- f"URL: { response .url } )
241+ 'Page Not Found Error. May be the result of an empty query. '
242+ + f'URL: { response .url }
243+ )
239244 elif response .status_code == 414 :
240245 _reason = response .reason
241246 _example = """
242- split_list = np.array_split(site_list, n) # n is number of chunks to divide query into \n
247+ # n is the number of chunks to divide the query into \n
248+ split_list = np.array_split(site_list, n)
243249 data_list = [] # list to store chunk results in \n
244250 # loop through chunks and make requests \n
245251 for site_list in split_list: \n
246- data = nwis.get_record(sites=site_list, service='dv', start=start, end=end) \n
252+ data = nwis.get_record(sites=site_list, service='dv', \n
253+ start=start, end=end) \n
247254 data_list.append(data) # append results to list"""
248255 raise ValueError (
249- " Request URL too long. Modify your query to use fewer sites. " +
250- f" API response reason: { _reason } " +
251- f" split your query: \n { _example } "
252- )
256+ ' Request URL too long. Modify your query to use fewer sites. '
257+ + f' API response reason: { _reason } '
258+ + f' split your query: \n { _example } '
259+ )
253260
254261 if response .text .startswith ('No sites/data' ):
255262 raise NoSitesError (response .url )
@@ -258,11 +265,13 @@ def query(url, payload, delimiter=',', ssl_check=True):
258265
259266
260267class NoSitesError (Exception ):
261- """Custom error class used when selection criteria returns no sites/data.
262- """
268+ """Custom error class used when selection criteria returns no sites/data."""
269+
263270 def __init__ (self , url ):
264271 self .url = url
265272
266273 def __str__ (self ):
267- return "No sites/data found using the selection criteria specified in url: {}" .format (self .url )
268-
274+ return (
275+ 'No sites/data found using the selection criteria specified in url: '
276+ '{url}'
277+ ).format (url = self .url )
0 commit comments