Brush up error handling, and make radQuestion_111 default to Green Pack if no answer is given

Author: asuffield

ef/nettask.py

@@ -129,8 +129,9 @@ def post(self, url, *args, **kwargs):
         parse_only = kwargs.pop('parse_only', None)
         return self._net_op(lambda url, *args, **kwargs: HTMLOp(qt_form_post(url, *args, **kwargs), timeout=timeout, parse_only=parse_only), url, *args, **kwargs)
 
-    def submit_form(self, form, user_fields={}, file=None, timeout=30, parse_only=None):
+    def submit_form(self, form, user_fields={}, file=None, timeout=30, parse_only=None, default_fields={}):
         fields = {}
+        fields_seen = set()
         action = form['action']
 
         for input in form.find_all('input'):
@@ -140,6 +141,7 @@ def submit_form(self, form, user_fields={}, file=None, timeout=30, parse_only=No
             if input.has_key('disabled'):
                 print "Skipping disabled select", name
                 continue
+            fields_seen.add(name)
             type = input['type'].lower()
             if type == 'image':
                 fields['%s.x' % name] = '1'
@@ -165,10 +167,18 @@ def submit_form(self, form, user_fields={}, file=None, timeout=30, parse_only=No
             if select.has_key('disabled'):
                 print "Skipping disabled select", name
                 continue
+            fields_seen.add(name)
             selected = filter(lambda o: o.has_key('selected'), select.find_all('option'))
             if len(selected):
                 fields[name] = selected[0]['value']
 
+        fields_missed = fields_seen - set(fields.keys())
+
+        for rexp in default_fields:
+            for name in fields_missed:
+                if rexp.match(name):
+                    fields[name] = default_fields[rexp]
+
         # Hideous eventsforce hack: they do this field-disabling mess in javascript
         for script in form.find_all('script'):
             m = re.search(r"depends = depends \+ '(.*)'", script.text)

ef/upload.py

@@ -32,7 +32,6 @@ def __init__(self, person, minimum_change, batch):
         self.batch = batch
         self.person = person
         self.photo = Photo.get(id=self.person.current_photo_id)
-        self.reply = None
         self.minimum_change = minimum_change
         self.skipped = False
         self.aborted = False
@@ -148,6 +147,7 @@ def task(self, image):
         while limit > 0 and not soup.find_all(['h1', 'h2', 'h3', 'h4'], text=re.compile(r'\s*Photo Upload\s*')):
             if not soup.form:
                 self.error.emit("Could not find form on registration pages (while looking for photo upload)")
+                return
             soup = yield self.submit_form(soup.form)
             error = soup.find('script', text=re.compile(r".*alert\('Error", re.S|re.I))
             if error:
@@ -156,15 +156,18 @@ def task(self, image):
                     msg = m.group(1)
                     msg = msg.replace(r'\r', '\r').replace(r'\n', '\n')
                     self.error.emit("Error from eventsforce: %s" % msg)
+                    return
                 else:
                     self.error.emit("Eventsforce generated an error, but it couldn't be recognised. Please try the web interface to see what's going on.")
+                    return
             limit = limit - 1
 
         if limit == 0:
             #f = open('tmp.html', 'w')
             #f.write(str(soup))
             #f.close()
             self.error.emit("Failed to find photo upload on registration pages")
+            return
 
         self.progress.emit(4)
 
@@ -196,16 +199,16 @@ def task(self, image):
 
         soup = yield self.submit_form(soup.form, {'uploadFile': '1', 'uploadTempPersonID': temp_person_id, 'uploadGuestNumber': guest_number, 'uploadDataID': data_id})
         self.progress.emit(6)
-        soup = yield self.submit_form(soup.form, {}, self.prepare_file_upload('FileStream', image))
+        soup = yield self.submit_form(soup.form, {}, file=self.prepare_file_upload('FileStream', image))
         self.progress.emit(7)
 
         for script in soup.find_all('script'):
             m = re.match(r'^\s*window\.location=\'(.*)\';', script.text)
             if m:
                 break
         if not m:
-            #print soup
             self.error.emit("Could not process photo upload (failed to find javascript refresh)")
+            return
         link = m.group(1)
 
         if re.search(r'File could not be saved', link):
@@ -242,17 +245,20 @@ def task(self, image):
         while limit > 0 and not re.search(r'Booking details', soup.find_all('h1')[1].text.strip(), re.I):
             if not soup.form:
                 self.error.emit("Could not find form on registration pages (while looking for final booking details page)")
-            soup = yield self.submit_form(soup.form)
+                erturn
+            soup = yield self.submit_form(soup.form, default_fields={re.compile('^radQuestion_111_'): 'Green Pack'})
             limit = limit - 1
 
         if limit == 0:
             self.error.emit("Failed to reach end of registration pages after uploading photo")
+            return
 
         self.progress.emit(11)
 
         final_proceed_button = soup.find('input', type='button', onclick=re.compile(r'gotoReceipt'))
         if not final_proceed_button:
             self.error.emit("Could not find final SAVE button")
+            return
         link = self.extract_link_from_silly_button(final_proceed_button)
         soup = yield self.get(link)
 
@@ -266,6 +272,7 @@ def task(self, image):
 
         if not re.search(r'Booking confirmation', soup.find_all('h1')[1].text, re.I):
             self.error.emit('Final page after upload did not look right, did something bad happen?')
+            return
 
         new_opinion = None
         if self.photo.opinion == 'ok':
@@ -311,7 +318,6 @@ def __init__(self):
         super(QtCore.QObject, self).__init__()
 
         self.tasks = None
-        self.reply = None
 
     @QtCore.pyqtSlot(dict, str, str)
     @catcherror

todo

@@ -8,6 +8,10 @@ We need to detect when a person is cancelled. This requires us to know
 what event we're working on. Make this a program-global state, rather
 than just a filter. Lots of things make more sense that way around.
 
+Improve fetcher reliability - detect photos which aren't found, for
+whatever reason
+
+
 High priority features
 ----------------------
 use pyinstaller instead of py2exe
@@ -29,3 +33,11 @@ Fix abuses of super() by getting rid of them. super() is insane
 Purge habitual getters/setters
 
 Multiple simultaneous uploads, for speed? Downloads? Priority of those?
+
+Better workflow: main window a timeline of previous/current/next,
+buttons to move back/forward more prominent, search as a thing you can
+open.
+
+Batch upload, report at the end on errors, don't stop after one error
+
+Button to abort current fetch/upload process