Merge branch 'develop' into bugfix/daemon-pid-recycling

Author: chrisjsewell

aiida/backends/djsite/db/migrations/0033_replace_text_field_with_json_field.py

@@ -16,6 +16,7 @@
 from django.db import migrations
 
 from aiida.backends.djsite.db.migrations import upgrade_schema_version
+from aiida.common.json import JSONEncoder
 
 REVISION = '1.0.33'
 DOWN_REVISION = '1.0.32'
@@ -32,27 +33,27 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='dbauthinfo',
             name='auth_params',
-            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict),
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, encoder=JSONEncoder),
         ),
         migrations.AlterField(
             model_name='dbauthinfo',
             name='metadata',
-            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict),
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, encoder=JSONEncoder),
         ),
         migrations.AlterField(
             model_name='dbcomputer',
             name='metadata',
-            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict),
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, encoder=JSONEncoder),
         ),
         migrations.AlterField(
             model_name='dbcomputer',
             name='transport_params',
-            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict),
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, encoder=JSONEncoder),
         ),
         migrations.AlterField(
             model_name='dblog',
             name='metadata',
-            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict),
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, encoder=JSONEncoder),
         ),
         upgrade_schema_version(REVISION, DOWN_REVISION)
     ]

aiida/backends/djsite/db/migrations/0037_attributes_extras_settings_json.py

@@ -19,6 +19,7 @@
 
 from aiida.backends.djsite.db.migrations import upgrade_schema_version
 from aiida.cmdline.utils import echo
+from aiida.common.json import JSONEncoder
 from aiida.common.timezone import datetime_to_isoformat
 
 REVISION = '1.0.37'
@@ -202,12 +203,12 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='dbnode',
             name='attributes',
-            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, null=True),
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, null=True, encoder=JSONEncoder),
         ),
         migrations.AddField(
             model_name='dbnode',
             name='extras',
-            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, null=True),
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, null=True, encoder=JSONEncoder),
         ),
         # Migrate the data from the DbAttribute table to the JSONB field
         migrations.RunPython(transition_attributes_extras, reverse_code=migrations.RunPython.noop),
@@ -232,7 +233,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='dbsetting',
             name='val',
-            field=django.contrib.postgres.fields.jsonb.JSONField(default=None, null=True),
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=None, null=True, encoder=JSONEncoder),
         ),
         # Migrate the data from the DbSetting EAV to the JSONB val field
         migrations.RunPython(transition_settings, reverse_code=migrations.RunPython.noop),

aiida/backends/djsite/db/migrations/0045_dbgroup_extras.py

@@ -11,7 +11,9 @@
 # pylint: disable=invalid-name
 import django.contrib.postgres.fields.jsonb
 from django.db import migrations
+
 from aiida.backends.djsite.db.migrations import upgrade_schema_version
+from aiida.common.json import JSONEncoder
 
 REVISION = '1.0.45'
 DOWN_REVISION = '1.0.44'
@@ -27,7 +29,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='dbgroup',
             name='extras',
-            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, null=False),
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, null=False, encoder=JSONEncoder),
         ),
         upgrade_schema_version(REVISION, DOWN_REVISION),
     ]

aiida/backends/djsite/db/migrations/0046_add_node_repository_metadata.py

@@ -13,7 +13,9 @@
 # pylint: disable=no-name-in-module,import-error
 import django.contrib.postgres.fields.jsonb
 from django.db import migrations
+
 from aiida.backends.djsite.db.migrations import upgrade_schema_version
+from aiida.common.json import JSONEncoder
 
 REVISION = '1.0.46'
 DOWN_REVISION = '1.0.45'
@@ -30,7 +32,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='dbnode',
             name='repository_metadata',
-            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, null=True),
+            field=django.contrib.postgres.fields.jsonb.JSONField(default=dict, null=True, encoder=JSONEncoder),
         ),
         upgrade_schema_version(REVISION, DOWN_REVISION),
     ]

aiida/backends/djsite/db/models.py

@@ -18,6 +18,7 @@
 
 import aiida.backends.djsite.db.migrations as migrations
 from aiida.common import timezone
+from aiida.common.json import JSONEncoder
 from aiida.common.utils import get_new_uuid
 
 # This variable identifies the schema version of this file.
@@ -124,10 +125,10 @@ class DbNode(m.Model):
     dbcomputer = m.ForeignKey('DbComputer', null=True, on_delete=m.PROTECT, related_name='dbnodes')
 
     # JSON Attributes
-    attributes = JSONField(default=dict, null=True)
+    attributes = JSONField(default=dict, null=True, encoder=JSONEncoder)
     # JSON Extras
-    extras = JSONField(default=dict, null=True)
-    repository_metadata = JSONField(default=dict, null=True)
+    extras = JSONField(default=dict, null=True, encoder=JSONEncoder)
+    repository_metadata = JSONField(default=dict, null=True, encoder=JSONEncoder)
 
     objects = m.Manager()
     # Return aiida Node instances or their subclasses instead of DbNode instances
@@ -184,7 +185,7 @@ def __str__(self):
 class DbSetting(m.Model):
     """This will store generic settings that should be database-wide."""
     key = m.CharField(max_length=1024, db_index=True, blank=False, unique=True)
-    val = JSONField(default=None, null=True)
+    val = JSONField(default=None, null=True, encoder=JSONEncoder)
     # I also add a description field for the variables
     description = m.TextField(blank=True)
     # Modification time of this attribute
@@ -256,7 +257,7 @@ class DbGroup(m.Model):
     # the groups
     user = m.ForeignKey(DbUser, on_delete=m.CASCADE, related_name='dbgroups')
     # JSON Extras
-    extras = JSONField(default=dict, null=False)
+    extras = JSONField(default=dict, null=False, encoder=JSONEncoder)
 
     class Meta:
         unique_together = (('label', 'type_string'),)
@@ -303,7 +304,7 @@ class DbComputer(m.Model):
     description = m.TextField(blank=True)
     scheduler_type = m.CharField(max_length=255)
     transport_type = m.CharField(max_length=255)
-    metadata = JSONField(default=dict)
+    metadata = JSONField(default=dict, encoder=JSONEncoder)
 
     def __str__(self):
         return f'{self.label} ({self.hostname})'
@@ -317,12 +318,12 @@ class DbAuthInfo(m.Model):
     # Delete the DbAuthInfo if either the user or the computer are removed
     aiidauser = m.ForeignKey(DbUser, on_delete=m.CASCADE)
     dbcomputer = m.ForeignKey(DbComputer, on_delete=m.CASCADE)
-    auth_params = JSONField(default=dict)  # contains mainly the remoteuser and the private_key
+    auth_params = JSONField(default=dict, encoder=JSONEncoder)  # contains mainly the remoteuser and the private_key
 
     # The keys defined in the metadata of the DbAuthInfo will override the
     # keys with the same label defined in the DbComputer (using a dict.update()
     # call of python).
-    metadata = JSONField(default=dict)
+    metadata = JSONField(default=dict, encoder=JSONEncoder)
     # Whether this computer is enabled (user-level enabling feature)
     enabled = m.BooleanField(default=True)
 
@@ -361,7 +362,7 @@ class DbLog(m.Model):
     levelname = m.CharField(max_length=50, db_index=True)
     dbnode = m.ForeignKey(DbNode, related_name='dblogs', on_delete=m.CASCADE)
     message = m.TextField(blank=True)
-    metadata = JSONField(default=dict)
+    metadata = JSONField(default=dict, encoder=JSONEncoder)
 
     def __str__(self):
         return f'DbLog: {self.levelname} for node {self.dbnode.id}: {self.message}'

aiida/cmdline/commands/cmd_group.py

@@ -32,7 +32,7 @@ def verdi_group():
 def group_add_nodes(group, force, nodes):
     """Add nodes to a group."""
     if not force:
-        click.confirm(f'Do you really want to add {len(nodes)} nodes to Group<{group.label}>?', abort=True)
+        click.confirm(f'Do you really want to add {len(nodes)} nodes to {group}?', abort=True)
 
     group.add_nodes(nodes)
 
@@ -47,9 +47,6 @@ def group_remove_nodes(group, nodes, clear, force):
     """Remove nodes from a group."""
     from aiida.orm import QueryBuilder, Group, Node
 
-    label = group.label
-    klass = group.__class__.__name__
-
     if nodes and clear:
         echo.echo_critical(
             'Specify either the `--clear` flag to remove all nodes or the identifiers of the nodes you want to remove.'
@@ -67,18 +64,18 @@ def group_remove_nodes(group, nodes, clear, force):
             group_node_pks = query.all(flat=True)
 
             if not group_node_pks:
-                echo.echo_critical(f'None of the specified nodes are in {klass}<{label}>.')
+                echo.echo_critical(f'None of the specified nodes are in {group}.')
 
             if len(node_pks) > len(group_node_pks):
                 node_pks = set(node_pks).difference(set(group_node_pks))
-                echo.echo_warning(f'{len(node_pks)} nodes with PK {node_pks} are not in {klass}<{label}>.')
+                echo.echo_warning(f'{len(node_pks)} nodes with PK {node_pks} are not in {group}.')
 
-            message = f'Are you sure you want to remove {len(group_node_pks)} nodes from {klass}<{label}>?'
+            message = f'Are you sure you want to remove {len(group_node_pks)} nodes from {group}?'
 
         elif clear:
-            message = f'Are you sure you want to remove ALL the nodes from {klass}<{label}>?'
+            message = f'Are you sure you want to remove ALL the nodes from {group}?'
         else:
-            echo.echo_critical(f'No nodes were provided for removal from {klass}<{label}>.')
+            echo.echo_critical(f'No nodes were provided for removal from {group}.')
 
         click.confirm(message, abort=True)
 
@@ -104,24 +101,21 @@ def group_delete(group, delete_nodes, dry_run, force, verbose, **traversal_rules
     from aiida.tools import delete_group_nodes, DELETE_LOGGER
     from aiida import orm
 
-    label = group.label
-    klass = group.__class__.__name__
-
     verbosity = logging.DEBUG if verbose else logging.INFO
     DELETE_LOGGER.setLevel(verbosity)
 
     if not (force or dry_run):
-        click.confirm(f'Are you sure to delete {klass}<{label}>?', abort=True)
+        click.confirm(f'Are you sure you want to delete {group}?', abort=True)
     elif dry_run:
-        echo.echo_info(f'Would have deleted {klass}<{label}>.')
+        echo.echo_info(f'Would have deleted {group}.')
 
     if delete_nodes:
 
         def _dry_run_callback(pks):
             if not pks or force:
                 return False
             echo.echo_warning(f'YOU ARE ABOUT TO DELETE {len(pks)} NODES! THIS CANNOT BE UNDONE!')
-            return not click.confirm('Shall I continue?', abort=True)
+            return not click.confirm('Do you want to continue?', abort=True)
 
         with override_log_formatter_context('%(message)s'):
             _, nodes_deleted = delete_group_nodes([group.pk], dry_run=dry_run or _dry_run_callback, **traversal_rules)
@@ -130,8 +124,9 @@ def _dry_run_callback(pks):
             return
 
     if not dry_run:
+        group_str = str(group)
         orm.Group.objects.delete(group.pk)
-        echo.echo_success(f'{klass}<{label}> deleted.')
+        echo.echo_success(f'{group_str} deleted.')
 
 
 @verdi_group.command('relabel')
@@ -143,9 +138,9 @@ def group_relabel(group, label):
     try:
         group.label = label
     except UniquenessError as exception:
-        echo.echo_critical(f'Error: {exception}.')
+        echo.echo_critical(str(exception))
     else:
-        echo.echo_success(f'Label changed to {label}')
+        echo.echo_success(f"Label changed to '{label}'")
 
 
 @verdi_group.command('description')
@@ -155,11 +150,11 @@ def group_relabel(group, label):
 def group_description(group, description):
     """Change the description of a group.
 
-    If no DESCRIPTION is defined, the current description will simply be echoed.
+    If no description is defined, the current description will simply be echoed.
     """
     if description:
         group.description = description
-        echo.echo_success(f'Changed the description of Group<{group.label}>')
+        echo.echo_success(f'Changed the description of {group}.')
     else:
         echo.echo(group.description)
 
@@ -172,7 +167,7 @@ def group_description(group, description):
     '--uuid',
     is_flag=True,
     default=False,
-    help='Show UUIDs together with PKs. Note: if the --raw option is also passed, PKs are not printed, but oly UUIDs.'
+    help='Show UUIDs together with PKs. Note: if the --raw option is also passed, PKs are not printed, but only UUIDs.'
 )
 @arguments.GROUP()
 @with_dbenv()
@@ -223,7 +218,7 @@ def group_show(group, raw, limit, uuid):
 
 @verdi_group.command('list')
 @options.ALL_USERS(help='Show groups for all users, rather than only for the current user.')
-@options.USER(help='Add a filter to show only groups belonging to a specific user')
+@options.USER(help='Add a filter to show only groups belonging to a specific user.')
 @options.ALL(help='Show groups of all types.')
 @options.TYPE_STRING()
 @click.option(
@@ -292,7 +287,7 @@ def group_list(
     if past_days:
         filters['time'] = {'>': timezone.now() - datetime.timedelta(days=past_days)}
 
-    # Query for specific group names
+    # Query for specific group labels
     filters['or'] = []
     if startswith:
         filters['or'].append({'label': {'like': f'{escape_for_sql_like(startswith)}%'}})
@@ -346,10 +341,10 @@ def group_list(
         table.append([projection_lambdas[field](group[0]) for field in projection_fields])
 
     if not all_entries:
-        echo.echo_info('to show groups of all types, use the `-a/--all` option.')
+        echo.echo_info('To show groups of all types, use the `-a/--all` option.')
 
     if not table:
-        echo.echo_info('no groups found matching the specified criteria.')
+        echo.echo_info('No groups found matching the specified criteria.')
     else:
         echo.echo(tabulate(table, headers=projection_header))
 
@@ -358,15 +353,15 @@ def group_list(
 @click.argument('group_label', nargs=1, type=click.STRING)
 @with_dbenv()
 def group_create(group_label):
-    """Create an empty group with a given name."""
+    """Create an empty group with a given label."""
     from aiida import orm
 
     group, created = orm.Group.objects.get_or_create(label=group_label)
 
     if created:
-        echo.echo_success(f"Group created with PK = {group.id} and name '{group.label}'")
+        echo.echo_success(f"Group created with PK = {group.pk} and label '{group.label}'.")
     else:
-        echo.echo_info(f"Group '{group.label}' already exists, PK = {group.id}")
+        echo.echo_info(f"Group with label '{group.label}' already exists: {group}.")
 
 
 @verdi_group.command('copy')
@@ -384,12 +379,12 @@ def group_copy(source_group, destination_group):
 
     # Issue warning if destination group is not empty and get user confirmation to continue
     if not created and not dest_group.is_empty:
-        echo.echo_warning(f'Destination group<{dest_group.label}> already exists and is not empty.')
+        echo.echo_warning(f'Destination {dest_group} already exists and is not empty.')
         click.confirm('Do you wish to continue anyway?', abort=True)
 
     # Copy nodes
     dest_group.add_nodes(list(source_group.nodes))
-    echo.echo_success(f'Nodes copied from group<{source_group.label}> to group<{dest_group.label}>')
+    echo.echo_success(f'Nodes copied from {source_group} to {dest_group}.')
 
 
 @verdi_group.group('path')