Disregard old test repository quirks for migration (#4954)

An old test created a `.gitignore` file in the Node repository folder
`path`, while also creating files under a `raw_input` folder.
This adds a logic test for this specific edge-case, migrating the
`raw_input` folder and ignoring the `path` folder for the specific Node.

Fixes #4910

Author: CasperWA

aiida/backends/general/migrations/utils.py

@@ -252,12 +252,16 @@ def get_node_repository_dirpaths(basepath, shard=None):
                 path = None
 
                 if 'path' in subdirs and 'raw_input' in subdirs:
-                    # If the `path` is empty, we simply ignore and set `raw_input` to be migrated, otherwise we add
-                    # the entry to `contains_both` which will cause the migration to fail.
-                    if os.listdir(dirpath / 'path'):
-                        contains_both.append(str(dirpath))
-                    else:
+                    # If the `path` folder is empty OR it contains *only* a `.gitignore`, we simply ignore and set
+                    # `raw_input` to be migrated, otherwise we add the entry to `contains_both` which will cause the
+                    # migration to fail.
+                    # See issue #4910 (https://github.com/aiidateam/aiida-core/issues/4910) for more information on the
+                    # `.gitignore` case.
+                    path_contents = os.listdir(dirpath / 'path')
+                    if not path_contents or path_contents == ['.gitignore']:
                         path = dirpath / 'raw_input'
+                    else:
+                        contains_both.append(str(dirpath))
                 elif 'path' in subdirs:
                     path = dirpath / 'path'
                 elif 'raw_input' in subdirs:

tests/backends/aiida_django/migrations/test_migrations_0047_migrate_repository.py

@@ -10,6 +10,7 @@
 # pylint: disable=import-error,no-name-in-module,invalid-name
 """Test migration of the old file repository to the disk object store."""
 import hashlib
+import os
 
 from aiida.backends.general.migrations import utils
 from .test_migrations_common import TestMigrations
@@ -33,14 +34,25 @@ def setUpBeforeMigration(self):
         dbnode_02.save()
         dbnode_03 = DbNode(user_id=self.default_user.id)
         dbnode_03.save()
+        dbnode_04 = DbNode(user_id=self.default_user.id)
+        dbnode_04.save()
 
         self.node_01_pk = dbnode_01.pk
         self.node_02_pk = dbnode_02.pk
         self.node_03_pk = dbnode_03.pk
+        self.node_04_pk = dbnode_04.pk
 
         utils.put_object_from_string(dbnode_01.uuid, 'sub/path/file_b.txt', 'b')
         utils.put_object_from_string(dbnode_01.uuid, 'sub/file_a.txt', 'a')
         utils.put_object_from_string(dbnode_02.uuid, 'output.txt', 'output')
+        utils.put_object_from_string(dbnode_04.uuid, '.gitignore', 'test')
+
+        # If both `path` and `raw_input` subfolders are present and `.gitignore` is in `path`, it should be ignored.
+        # Cannot use `put_object_from_string` here as it statically writes under the `path` folder.
+        raw_input_sub_folder = utils.get_node_repository_sub_folder(dbnode_04.uuid, subfolder='raw_input')
+        os.makedirs(raw_input_sub_folder, exist_ok=True)
+        with open(os.path.join(raw_input_sub_folder, 'input.txt'), 'w', encoding='utf-8') as handle:
+            handle.write('input')
 
         # When multiple migrations are ran, it is possible that migration 0047 is run at a point where the repository
         # container does not have a UUID (at that point in the migration) and so the setting gets set to `None`. This
@@ -56,6 +68,7 @@ def test_migration(self):
         node_01 = DbNode.objects.get(pk=self.node_01_pk)
         node_02 = DbNode.objects.get(pk=self.node_02_pk)
         node_03 = DbNode.objects.get(pk=self.node_03_pk)
+        node_04 = DbNode.objects.get(pk=self.node_04_pk)
 
         assert node_01.repository_metadata == {
             'o': {
@@ -83,11 +96,19 @@ def test_migration(self):
             }
         }
         assert node_03.repository_metadata == {}
+        assert node_04.repository_metadata == {
+            'o': {
+                'input.txt': {
+                    'k': hashlib.sha256('input'.encode('utf-8')).hexdigest()
+                }
+            }
+        }
 
         for hashkey, content in (
             (node_01.repository_metadata['o']['sub']['o']['path']['o']['file_b.txt']['k'], b'b'),
             (node_01.repository_metadata['o']['sub']['o']['file_a.txt']['k'], b'a'),
             (node_02.repository_metadata['o']['output.txt']['k'], b'output'),
+            (node_04.repository_metadata['o']['input.txt']['k'], b'input'),
         ):
             assert utils.get_repository_object(hashkey) == content
 

tests/backends/aiida_sqlalchemy/test_migrations.py

@@ -1816,29 +1816,44 @@ def setUpBeforeMigration(self):
                 node_02 = DbNode(user_id=default_user.id, uuid=get_new_uuid())
                 node_03 = DbNode(user_id=default_user.id, uuid=get_new_uuid())
                 node_04 = DbNode(user_id=default_user.id, uuid=get_new_uuid())
+                node_05 = DbNode(user_id=default_user.id, uuid=get_new_uuid())
 
                 session.add(node_01)
                 session.add(node_02)
                 session.add(node_03)  # Empty repository folder
                 session.add(node_04)  # Both `path` and `raw_input` subfolder
+                session.add(node_05)  # Both `path` and `raw_input` subfolder & `.gitignore` in `path`
                 session.commit()
 
                 assert node_01.uuid is not None
                 assert node_02.uuid is not None
                 assert node_03.uuid is not None
                 assert node_04.uuid is not None
+                assert node_05.uuid is not None
 
                 self.node_01_pk = node_01.id
                 self.node_02_pk = node_02.id
                 self.node_03_pk = node_03.id
                 self.node_04_pk = node_04.id
+                self.node_05_pk = node_05.id
 
                 utils.put_object_from_string(node_01.uuid, 'sub/path/file_b.txt', 'b')
                 utils.put_object_from_string(node_01.uuid, 'sub/file_a.txt', 'a')
                 utils.put_object_from_string(node_02.uuid, 'output.txt', 'output')
 
                 os.makedirs(utils.get_node_repository_sub_folder(node_04.uuid, 'path'), exist_ok=True)
                 os.makedirs(utils.get_node_repository_sub_folder(node_04.uuid, 'raw_input'), exist_ok=True)
+                os.makedirs(utils.get_node_repository_sub_folder(node_05.uuid, 'path'), exist_ok=True)
+                os.makedirs(utils.get_node_repository_sub_folder(node_05.uuid, 'raw_input'), exist_ok=True)
+
+                utils.put_object_from_string(node_05.uuid, '.gitignore', 'test')
+                with open(
+                    os.path.join(
+                        utils.get_node_repository_sub_folder(node_05.uuid, 'raw_input'), 'input.txt'),
+                        'w',
+                        encoding='utf-8',
+                ) as handle:
+                    handle.write('input')
 
                 # Add a repository folder for a node that no longer exists - i.e. it may have been deleted.
                 utils.put_object_from_string(get_new_uuid(), 'file_of_deleted_node', 'output')
@@ -1859,6 +1874,7 @@ def test_migration(self):
                 node_01 = session.query(DbNode).filter(DbNode.id == self.node_01_pk).one()
                 node_02 = session.query(DbNode).filter(DbNode.id == self.node_02_pk).one()
                 node_03 = session.query(DbNode).filter(DbNode.id == self.node_03_pk).one()
+                node_05 = session.query(DbNode).filter(DbNode.id == self.node_05_pk).one()
 
                 assert node_01.repository_metadata == {
                     'o': {
@@ -1886,11 +1902,19 @@ def test_migration(self):
                     }
                 }
                 assert node_03.repository_metadata == {}
+                assert node_05.repository_metadata == {
+                    'o': {
+                        'input.txt': {
+                            'k': hashlib.sha256('input'.encode('utf-8')).hexdigest()
+                        }
+                    }
+                }
 
                 for hashkey, content in (
                     (node_01.repository_metadata['o']['sub']['o']['path']['o']['file_b.txt']['k'], b'b'),
                     (node_01.repository_metadata['o']['sub']['o']['file_a.txt']['k'], b'a'),
                     (node_02.repository_metadata['o']['output.txt']['k'], b'output'),
+                    (node_05.repository_metadata['o']['input.txt']['k'], b'input'),
                 ):
                     assert utils.get_repository_object(hashkey) == content