Merge pull request #607 from alephdata/feature/disable-sentry-processing-exceptions

Introduce a setting to disable sending ProcessingExceptions to Sentry

Author: stchris

ingestors/manager.py

@@ -219,7 +219,8 @@ def ingest(self, file_path, entity, **kwargs):
             log.exception(f"[{repr(entity)}] Failed to process: {pexc}")
             INGESTIONS_FAILED.labels(ingestor=ingestor_name).inc()
             entity.set("processingError", stringify(pexc))
-            capture_exception(pexc)
+            if settings.SENTRY_CAPTURE_PROCESSING_EXCEPTIONS:
+                capture_exception(pexc)
         finally:
             self.finalize(entity)
 

ingestors/settings.py

@@ -50,3 +50,10 @@
 
 # Also store cached values in the SQL database
 sls.TAGS_DATABASE_URI = fts.DATABASE_URI
+
+# ProcessingException is thrown whenever something goes wrong wiht
+# parsing a file. Enable this with care, it can easily eat up the
+# Sentry quota of events.
+SENTRY_CAPTURE_PROCESSING_EXCEPTIONS = env.to_bool(
+    "SENTRY_CAPTURE_PROCESSING_EXCEPTIONS", False
+)