Merge pull request #551 from alephdata/bug/aleph-2957

Emit a verbose error when processing a password-protected XLS / XLSX file

Author: catileptic

ingestors/documents/pdf.py

@@ -2,7 +2,7 @@
 
 from ingestors.ingestor import Ingestor
 from ingestors.support.pdf import PDFSupport
-from ingestors.exc import ProcessingException, UnauthorizedError
+from ingestors.exc import ProcessingException, UnauthorizedError, ENCRYPTED_MSG
 
 log = logging.getLogger(__name__)
 
@@ -24,9 +24,7 @@ def ingest(self, file_path, entity):
         try:
             self.parse_and_ingest(file_path, entity, self.manager)
         except UnauthorizedError as pwe:
-            raise ProcessingException(
-                "Could not extract PDF file. The PDF is protected with a password. Try removing the password protection and re-uploading the documents."
-            ) from pwe
+            raise ProcessingException(ENCRYPTED_MSG) from pwe
         except Exception as ex:
             raise ProcessingException("Could not extract PDF file: %r" % ex) from ex
 

ingestors/exc.py

@@ -1,3 +1,6 @@
+ENCRYPTED_MSG = "The document might be protected with a password. Try removing the password protection and re-uploading the documents."
+
+
 class ProcessingException(Exception):
     "A data-related error occuring during file processing."
     pass

ingestors/manager.py

@@ -17,7 +17,7 @@
 from followthemoney.namespace import Namespace
 
 from ingestors.directory import DirectoryIngestor
-from ingestors.exc import ProcessingException
+from ingestors.exc import ProcessingException, ENCRYPTED_MSG
 from ingestors.util import filter_text, remove_directory
 from ingestors import settings
 
@@ -94,6 +94,9 @@ def auction(self, file_path, entity):
                 return DirectoryIngestor
             entity.add("mimeType", self.MAGIC.from_file(file_path.as_posix()))
 
+        if "application/encrypted" in entity.get("mimeType"):
+            raise ProcessingException(ENCRYPTED_MSG)
+
         best_score, best_cls = 0, None
         for cls in get_extensions("ingestors"):
             score = cls.match(file_path, entity)

ingestors/support/ooxml.py

@@ -4,6 +4,7 @@
 
 from ingestors.support.xml import XMLSupport
 from ingestors.support.timestamp import TimestampSupport
+from ingestors.exc import ProcessingException, ENCRYPTED_MSG
 
 # from ingestors.exc import ProcessingException
 
@@ -53,6 +54,9 @@ def get(ns, name):
     @classmethod
     def inspect_ooxml_manifest(cls, file_path):
         if not zipfile.is_zipfile(file_path):
+            # password-protected Excel files are detected as zipfiles
+            if "Excel" in cls.__name__:
+                raise ProcessingException(ENCRYPTED_MSG)
             return False
         try:
             with zipfile.ZipFile(file_path, "r") as zf:

ingestors/tabular/xls.py

@@ -7,7 +7,7 @@
 from ingestors.ingestor import Ingestor
 from ingestors.support.table import TableSupport
 from ingestors.support.ole import OLESupport
-from ingestors.exc import ProcessingException
+from ingestors.exc import ProcessingException, ENCRYPTED_MSG
 
 log = logging.getLogger(__name__)
 
@@ -49,6 +49,8 @@ def ingest(self, file_path, entity):
         self.extract_ole_metadata(file_path, entity)
         try:
             book = xlrd.open_workbook(file_path, formatting_info=False)
+        except XLRDError:
+            raise ProcessingException(ENCRYPTED_MSG)
         except Exception as err:
             raise ProcessingException("Invalid Excel file: %s" % err) from err
 

ingestors/tabular/xlsx.py

@@ -1,4 +1,5 @@
 import logging
+
 from followthemoney import model
 from openpyxl import load_workbook
 from xml.etree.ElementTree import ParseError

tests/fixtures/password_protected.xls

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 from .support import TestCase
+from ingestors.exc import ENCRYPTED_MSG
 
 
 class PDFIngestorTest(TestCase):
@@ -84,10 +85,7 @@ def test_ingest_protected(self):
         text = self.manager.entities[0].first("bodyText")
         self.assertEqual(None, text)
         err = self.manager.entities[0].first("processingError")
-        self.assertIn(
-            "Could not extract PDF file. The PDF is protected with a password. Try removing the password protection and re-uploading the documents.",
-            err,
-        )
+        self.assertIn(ENCRYPTED_MSG, err)
         status = self.manager.entities[0].first("processingStatus")
         self.assertEqual("failure", status)
 

tests/fixtures/password_protected.xlsx

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from .support import TestCase
+from ingestors.exc import ENCRYPTED_MSG
 
 
 class TabularIngestorTest(TestCase):
@@ -34,3 +35,21 @@ def test_unicode_ods(self):
         tables = [t.first("title") for t in tables]
         self.assertIn("Лист1", tables)
         self.assertEqual(entity.schema.name, "Workbook")
+
+    def test_password_protected_xlsx(self):
+        fixture_path, entity = self.fixture("password_protected.xlsx")
+        self.manager.ingest(fixture_path, entity)
+        self.assertEqual(len(self.get_emitted()), 1)
+        err = self.manager.entities[0].first("processingError")
+        self.assertIn(ENCRYPTED_MSG, err)
+        status = self.manager.entities[0].first("processingStatus")
+        self.assertEqual("failure", status)
+
+    def test_password_protected_xls(self):
+        fixture_path, entity = self.fixture("password_protected.xls")
+        self.manager.ingest(fixture_path, entity)
+        self.assertEqual(len(self.get_emitted()), 1)
+        err = self.manager.entities[0].first("processingError")
+        self.assertIn(ENCRYPTED_MSG, err)
+        status = self.manager.entities[0].first("processingStatus")
+        self.assertEqual("failure", status)