Merge branch 'dev'

breccia_mapper/settings.py

@@ -38,7 +38,7 @@ The most likely required settings are: SECRET_KEY, DEBUG, ALLOWED_HOSTS, DATABAS
 - DATABASE_URL
   default: sqlite://db.sqlite3
   URL to database - uses format described at https://github.com/jacobian/dj-database-url
-  
+
 - DBBACKUP_STORAGE_LOCATION
   default: .dbbackup
   Directory where database backups should be stored
@@ -106,7 +106,6 @@ from django.urls import reverse_lazy
 from decouple import config, Csv
 import dj_database_url
 
-
 # Settings exported to templates
 # https://github.com/jakubroztocil/django-settings-export
 
@@ -116,15 +115,12 @@ SETTINGS_EXPORT = [
     'PROJECT_SHORT_NAME',
 ]
 
-
 PROJECT_LONG_NAME = config('PROJECT_LONG_NAME', default='Project Long Name')
 PROJECT_SHORT_NAME = config('PROJECT_SHORT_NAME', default='shortname')
 
-
 # Build paths inside the project like this: BASE_DIR.joinpath(...)
 BASE_DIR = pathlib.Path(__file__).parent.parent
 
-
 # SECURITY WARNING: keep the secret key used in production secret!
 SECRET_KEY = config('SECRET_KEY')
 
@@ -134,9 +130,7 @@ DEBUG = config('DEBUG', default=False, cast=bool)
 ALLOWED_HOSTS = config(
     'ALLOWED_HOSTS',
     default='*' if DEBUG else '127.0.0.1,localhost,localhost.localdomain',
-    cast=Csv()
+    cast=Csv())
-)
-
 
 # Application definition
 
@@ -157,6 +151,7 @@ THIRD_PARTY_APPS = [
     'django_countries',
     'django_select2',
     'rest_framework',
+    'post_office',
 ]
 
 FIRST_PARTY_APPS = [
@@ -199,16 +194,14 @@ TEMPLATES = [
 
 WSGI_APPLICATION = 'breccia_mapper.wsgi.application'
 
-
 # Database
 # https://docs.djangoproject.com/en/2.2/ref/settings/#databases
 
 DATABASES = {
-    'default': config(
+    'default':
-        'DATABASE_URL',
+    config('DATABASE_URL',
-        default='sqlite:///' + str(BASE_DIR.joinpath('db.sqlite3')),
+           default='sqlite:///' + str(BASE_DIR.joinpath('db.sqlite3')),
-        cast=dj_database_url.parse
+           cast=dj_database_url.parse)
-    )
 }
 
 # Django DBBackup
@@ -216,10 +209,11 @@ DATABASES = {
 
 DBBACKUP_STORAGE = 'django.core.files.storage.FileSystemStorage'
 DBBACKUP_STORAGE_OPTIONS = {
-    'location': config('DBBACKUP_STORAGE_LOCATION', default=BASE_DIR.joinpath('.dbbackup')),
+    'location':
+    config('DBBACKUP_STORAGE_LOCATION',
+           default=BASE_DIR.joinpath('.dbbackup')),
 }
 
-
 # Django REST Framework
 # https://www.django-rest-framework.org/
 
@@ -234,22 +228,25 @@ REST_FRAMEWORK = {
     ],
 }
 
-
 # Password validation
 # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
 
 AUTH_PASSWORD_VALIDATORS = [
     {
-        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+        'NAME':
+        'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
     },
     {
-        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+        'NAME':
+        'django.contrib.auth.password_validation.MinimumLengthValidator',
     },
     {
-        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+        'NAME':
+        'django.contrib.auth.password_validation.CommonPasswordValidator',
     },
     {
-        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+        'NAME':
+        'django.contrib.auth.password_validation.NumericPasswordValidator',
     },
 ]
 
@@ -264,7 +261,6 @@ LOGIN_URL = reverse_lazy('login')
 
 LOGIN_REDIRECT_URL = reverse_lazy('index')
 
-
 # Internationalization
 # https://docs.djangoproject.com/en/2.2/topics/i18n/
 
@@ -278,7 +274,6 @@ USE_L10N = True
 
 USE_TZ = True
 
-
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/2.2/howto/static-files/
 
@@ -286,10 +281,7 @@ STATIC_URL = '/static/'
 
 STATIC_ROOT = BASE_DIR.joinpath('static')
 
-STATICFILES_DIRS = [
+STATICFILES_DIRS = [BASE_DIR.joinpath('breccia_mapper', 'static')]
-    BASE_DIR.joinpath('breccia_mapper', 'static')
-]
-
 
 # Logging - NB the logger name is empty to capture all output
 
@@ -332,12 +324,14 @@ LOGGING_CONFIG = None
 logging.config.dictConfig(LOGGING)
 logger = logging.getLogger(__name__)
 
-
 # Admin panel variables
 
 CONSTANCE_CONFIG = collections.OrderedDict([
-    ('NOTICE_TEXT', ('', 'Text to be displayed in a notice banner at the top of every page.')),
+    ('NOTICE_TEXT',
-    ('NOTICE_CLASS', ('alert-warning', 'CSS class to use for background of notice banner.')),
+     ('',
+      'Text to be displayed in a notice banner at the top of every page.')),
+    ('NOTICE_CLASS', ('alert-warning',
+                      'CSS class to use for background of notice banner.')),
 ])
 
 CONSTANCE_CONFIG_FIELDSETS = {
@@ -346,7 +340,6 @@ CONSTANCE_CONFIG_FIELDSETS = {
 
 CONSTANCE_BACKEND = 'constance.backends.database.DatabaseBackend'
 
-
 # Bootstrap settings
 # See https://django-bootstrap4.readthedocs.io/en/latest/settings.html
 
@@ -354,37 +347,42 @@ BOOTSTRAP4 = {
     'include_jquery': 'full',
 }
 
-
 # Email backend settings
 # See https://docs.djangoproject.com/en/3.0/topics/email
 
 EMAIL_HOST = config('EMAIL_HOST', default=None)
-DEFAULT_FROM_EMAIL = config('DEFAULT_FROM_EMAIL', default=None)
+DEFAULT_FROM_EMAIL = config(
+    'DEFAULT_FROM_EMAIL',
+    default=f'{PROJECT_SHORT_NAME}@localhost.localdomain')
 SERVER_EMAIL = DEFAULT_FROM_EMAIL
 
 if EMAIL_HOST is None:
     EMAIL_BACKEND = 'django.core.mail.backends.filebased.EmailBackend'
-    EMAIL_FILE_PATH = config('EMAIL_FILE_PATH', default=str(BASE_DIR.joinpath('mail.log')))
+    EMAIL_FILE_PATH = config('EMAIL_FILE_PATH',
+                             default=str(BASE_DIR.joinpath('mail.log')))
 
 else:
     EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
     EMAIL_HOST_USER = config('EMAIL_HOST_USER', default=None)
     EMAIL_HOST_PASSWORD = config('EMAIL_HOST_PASSWORD', default=None)
-    
-    EMAIL_PORT = config('EMAIL_PORT', default=25, cast=int)
-    EMAIL_USE_TLS = config('EMAIL_USE_TLS', default=(EMAIL_PORT == 587), cast=bool)
-    EMAIL_USE_SSL = config('EMAIL_USE_SSL', default=(EMAIL_PORT == 465), cast=bool)
 
+    EMAIL_PORT = config('EMAIL_PORT', default=25, cast=int)
+    EMAIL_USE_TLS = config('EMAIL_USE_TLS',
+                           default=(EMAIL_PORT == 587),
+                           cast=bool)
+    EMAIL_USE_SSL = config('EMAIL_USE_SSL',
+                           default=(EMAIL_PORT == 465),
+                           cast=bool)
 
 # Import customisation app settings if present
 
+CUSTOMISATION_NAME = None
 TEMPLATE_NAME_INDEX = 'index.html'
+TEMPLATE_WELCOME_EMAIL_NAME = 'welcome-email'
 
 try:
-    from custom.settings import (
+    from custom.settings import (CUSTOMISATION_NAME, TEMPLATE_NAME_INDEX,
-        CUSTOMISATION_NAME,
+                                 TEMPLATE_WELCOME_EMAIL_NAME)
-        TEMPLATE_NAME_INDEX
-    )
     logger.info("Loaded customisation app: %s", CUSTOMISATION_NAME)
 
     INSTALLED_APPS.append('custom')

people/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'people.apps.PeopleConfig'

people/admin.py

@@ -8,7 +8,12 @@ from django.contrib.auth.admin import UserAdmin
 from . import models
 
 
-admin.site.register(models.User, UserAdmin)
+@admin.register(models.User)
+class CustomUserAdmin(UserAdmin):
+    """Add email address field to new user form."""
+    add_fieldsets = UserAdmin.add_fieldsets + (
+        ('Details', {'fields': ('email', )}),
+    )  # yapf: disable
 
 
 @admin.register(models.Organisation)

people/apps.py

@@ -1,5 +1,69 @@
+import logging
+
 from django.apps import AppConfig
+from django.conf import settings
+from django.core import serializers
+from django.db.models.signals import post_save
+
+logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
+
+
+def load_welcome_template_fixture(fixture_path) -> bool:
+    """Load welcome email template from a JSON fixture."""
+    try:
+        with open(fixture_path) as f:
+            for deserialized in serializers.deserialize('json', f):
+                if deserialized.object.name == settings.TEMPLATE_WELCOME_EMAIL_NAME:
+                    deserialized.save()
+                    logger.warning('Welcome email template \'%s\' loaded',
+                                   deserialized.object.name)
+                    return True
+
+            return False
+
+    except FileNotFoundError:
+        logger.warning('Email template fixture not found.')
+        return False
+
+
+def send_welcome_email(sender, instance, created, **kwargs):
+    from post_office import models
+
+    if not created:
+        # If user already exists, don't send welcome message
+        return
+
+    try:
+        instance.send_welcome_email()
+
+    except models.EmailTemplate.DoesNotExist:
+        logger.warning(
+            'Welcome email template \'%s\' not found - attempting to load from fixtures',
+            settings.TEMPLATE_WELCOME_EMAIL_NAME)
+
+        is_loaded = False
+        if settings.CUSTOMISATION_NAME:
+            # Customisation app present - try here first
+            is_loaded |= load_welcome_template_fixture(
+                settings.BASE_DIR.joinpath('custom', 'fixtures',
+                                           'email_templates.json'))
+
+        # |= operator shortcuts - only try here if we don't already have it
+        is_loaded |= load_welcome_template_fixture(
+            settings.BASE_DIR.joinpath('people', 'fixtures',
+                                       'email_templates.json'))
+
+        if is_loaded:
+            instance.send_welcome_email()
+
+        else:
+            logger.error('Welcome email template \'%s\' not found',
+                         settings.TEMPLATE_WELCOME_EMAIL_NAME)
 
 
 class PeopleConfig(AppConfig):
     name = 'people'
+
+    def ready(self) -> None:
+        # Activate signal handlers
+        post_save.connect(send_welcome_email, sender='people.user')

people/fixtures/email_templates.json

@@ -0,0 +1,16 @@
+[
+    {
+        "model": "post_office.emailtemplate",
+        "fields": {
+            "name": "welcome-email",
+            "description": "Default welcome email template",
+            "created": "2020-04-27T12:13:30.448Z",
+            "last_updated": "2020-04-27T14:45:27.152Z",
+            "subject": "Welcome to {{settings.PROJECT_LONG_NAME}}",
+            "content": "Dear {{ user.get_full_name }},\r\n\r\nWelcome to {{ settings.PROJECT_LONG_NAME }}.\r\n\r\nThanks,\r\n\r\nThe {{ settings.PROJECT_LONG_NAME }} team",
+            "html_content": "<h1>{{ settings.PROJECT_LONG_NAME }}</h1>\r\n\r\nDear {{ user.get_full_name }},\r\n\r\nWelcome to {{ settings.PROJECT_LONG_NAME }}.\r\n\r\nThanks,\r\n\r\nThe {{ settings.PROJECT_LONG_NAME }} team",
+            "language": "",
+            "default_template": null
+        }
+    }
+]

people/migrations/0018_require_user_email.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.10 on 2020-05-28 15:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('people', '0017_answerset_replaced_timestamp'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='email',
+            field=models.EmailField(max_length=254, verbose_name='email address'),
+        ),
+    ]

people/models/person.py

@@ -1,13 +1,20 @@
+import logging
+
 from django.conf import settings
 from django.contrib.auth.models import AbstractUser
+from django.core.exceptions import ValidationError
 from django.db import models
 from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
 
 from django_countries.fields import CountryField
+from django_settings_export import settings_export
+from post_office import mail
 
 from backports.db.models.enums import TextChoices
 
+logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
+
 __all__ = [
     'User',
     'Organisation',
@@ -22,30 +29,54 @@ class User(AbstractUser):
     """
     Custom user model in case we need to make changes later.
     """
+    email = models.EmailField(_('email address'), blank=False, null=False)
+
     def has_person(self) -> bool:
         """
         Does this user have a linked :class:`Person` record?
         """
         return hasattr(self, 'person')
 
+    def send_welcome_email(self):
+        """Send a welcome email to a new user."""
+        # Get exported data from settings.py first
+        context = settings_export(None)
+        context.update({
+            'user': self,
+        })
+
+        logger.info('Sending welcome mail to user \'%s\'', self.username)
+
+        try:
+            mail.send(
+                [self.email],
+                sender=settings.DEFAULT_FROM_EMAIL,
+                template=settings.TEMPLATE_WELCOME_EMAIL_NAME,
+                context=context,
+                priority='now'  # Send immediately - don't add to queue
+            )
+
+        except ValidationError:
+            logger.error(
+                'Sending welcome mail failed, invalid email for user \'%s\'',
+                self.username)
+
 
 class Organisation(models.Model):
     """
     Organisation to which a :class:`Person` belongs.
     """
-    name = models.CharField(max_length=255,
+    name = models.CharField(max_length=255, blank=False, null=False)
-                            blank=False, null=False)
+
-    
     def __str__(self) -> str:
         return self.name
-    
+
-    
+
 class Role(models.Model):
     """
     Role which a :class:`Person` holds within the project.
     """
-    name = models.CharField(max_length=255,
+    name = models.CharField(max_length=255, blank=False, null=False)
-                            blank=False, null=False)
 
     def __str__(self) -> str:
         return self.name
@@ -55,23 +86,20 @@ class Discipline(models.Model):
     """
     Discipline within which a :class:`Person` works.
     """
-    name = models.CharField(max_length=255,
+    name = models.CharField(max_length=255, blank=False, null=False)
-                            blank=False, null=False)
 
     #: Short code using system such as JACS 3
-    code = models.CharField(max_length=15,
+    code = models.CharField(max_length=15, blank=True, null=False)
-                            blank=True, null=False)
 
     def __str__(self) -> str:
         return self.name
-    
+
-    
+
 class Theme(models.Model):
     """
     Project theme within which a :class:`Person` works.
     """
-    name = models.CharField(max_length=255,
+    name = models.CharField(max_length=255, blank=False, null=False)
-                            blank=False, null=False)
 
     def __str__(self) -> str:
         return self.name
@@ -88,21 +116,22 @@ class Person(models.Model):
     user = models.OneToOneField(settings.AUTH_USER_MODEL,
                                 related_name='person',
                                 on_delete=models.CASCADE,
-                                blank=True, null=True)
+                                blank=True,
+                                null=True)
 
     #: Name of the person
-    name = models.CharField(max_length=255,
+    name = models.CharField(max_length=255, blank=False, null=False)
-                            blank=False, null=False)
 
     #: Is this person a member of the core project team?
-    core_member = models.BooleanField(default=False,
+    core_member = models.BooleanField(default=False, blank=False, null=False)
-                                      blank=False, null=False)
 
     #: People with whom this person has relationship - via intermediate :class:`Relationship` model
-    relationship_targets = models.ManyToManyField('self', related_name='relationship_sources',
+    relationship_targets = models.ManyToManyField(
-                                                  through='Relationship',
+        'self',
-                                                  through_fields=('source', 'target'),
+        related_name='relationship_sources',
-                                                  symmetrical=False)
+        through='Relationship',
+        through_fields=('source', 'target'),
+        symmetrical=False)
 
     ###############################################################
     # Data collected for analysis of community makeup and structure
@@ -115,7 +144,8 @@ class Person(models.Model):
 
     gender = models.CharField(max_length=1,
                               choices=GenderChoices.choices,
-                              blank=True, null=False)
+                              blank=True,
+                              null=False)
 
     class AgeGroupChoices(TextChoices):
         LTE_25 = '<=25', _('25 or under')
@@ -131,8 +161,9 @@ class Person(models.Model):
 
     age_group = models.CharField(max_length=5,
                                  choices=AgeGroupChoices.choices,
-                                 blank=True, null=False)
+                                 blank=True,
-    
+                                 null=False)
+
     nationality = CountryField(blank=True, null=True)
 
     country_of_residence = CountryField(blank=True, null=True)
@@ -141,34 +172,33 @@ class Person(models.Model):
     organisation = models.ForeignKey(Organisation,
                                      on_delete=models.PROTECT,
                                      related_name='members',
-                                     blank=True, null=True)
+                                     blank=True,
+                                     null=True)
 
     #: Job title this person holds within their organisation
-    job_title = models.CharField(max_length=255,
+    job_title = models.CharField(max_length=255, blank=True, null=False)
-                                 blank=True, null=False)
 
     #: Discipline within which this person works
     discipline = models.ForeignKey(Discipline,
                                    on_delete=models.PROTECT,
                                    related_name='people',
-                                   blank=True, null=True)
+                                   blank=True,
+                                   null=True)
 
     #: Role this person holds within the project
     role = models.ForeignKey(Role,
                              on_delete=models.PROTECT,
                              related_name='holders',
-                             blank=True, null=True)
+                             blank=True,
+                             null=True)
 
     #: Project themes within this person works
-    themes = models.ManyToManyField(Theme,
+    themes = models.ManyToManyField(Theme, related_name='people', blank=True)
-                                    related_name='people',
-                                    blank=True)
 
     @property
     def relationships(self):
         return self.relationships_as_source.all().union(
-            self.relationships_as_target.all()
+            self.relationships_as_target.all())
-        )
 
     def get_absolute_url(self):
         return reverse('people:person.detail', kwargs={'pk': self.pk})

people/views/network.py

@@ -2,15 +2,18 @@
 Views for displaying networks of :class:`People` and :class:`Relationship`s.
 """
 
+import logging
+
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.db.models import Q
 from django.forms import ValidationError
 from django.utils import timezone
 from django.views.generic import FormView
 
-
 from people import forms, models, serializers
 
+logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
+
 
 class NetworkView(LoginRequiredMixin, FormView):
     """
@@ -47,35 +50,45 @@ class NetworkView(LoginRequiredMixin, FormView):
         if not at_date:
             at_date = timezone.now().date()
 
+        # Filter on timestamp__date doesn't seem to work on MySQL
+        # To compare datetimes we need at_date to be midnight at
+        # the *end* of the day in question - so add one day here
+        at_date += timezone.timedelta(days=1)
+
         relationship_answerset_set = models.RelationshipAnswerSet.objects.filter(
-            Q(replaced_timestamp__date__gte=at_date) | Q(replaced_timestamp__isnull=True),
+            Q(replaced_timestamp__gte=at_date)
-            timestamp__date__lte=at_date
+            | Q(replaced_timestamp__isnull=True),
-        )
+            timestamp__lte=at_date)
+
+        logger.info('Found %d relationship answer sets for %s',
+                    relationship_answerset_set.count(), at_date)
 
         # Filter answers to relationship questions
         for field, values in form.cleaned_data.items():
             if field.startswith('question_') and values:
                 relationship_answerset_set = relationship_answerset_set.filter(
-                    question_answers__in=values
+                    question_answers__in=values)
-                )
+
+        logger.info('Found %d relationship answer sets matching filters',
+                    relationship_answerset_set.count())
 
         context['person_set'] = serializers.PersonSerializer(
-            models.Person.objects.all(),
+            models.Person.objects.all(), many=True).data
-            many=True
-        ).data
 
         context['relationship_set'] = serializers.RelationshipSerializer(
             models.Relationship.objects.filter(
-                pk__in=relationship_answerset_set.values_list('relationship', flat=True)
+                pk__in=relationship_answerset_set.values_list('relationship',
-            ),
+                                                              flat=True)),
-            many=True
+            many=True).data
-        ).data
+
+        logger.info('Found %d distinct relationships matching filters',
+                    len(context['relationship_set']))
 
         return context
 
     def form_valid(self, form):
         try:
             return self.render_to_response(self.get_context_data())
-        
+
         except ValidationError:
             return self.form_invalid(form)

playbook.yml

@@ -16,5 +16,3 @@
 
   vars:
     ansible_python_interpreter: python2
-    db_user: 'breccia'
-    db_pass: 'breccia'

requirements.txt

@@ -9,11 +9,13 @@ django-countries==5.5
 django-dbbackup==3.2.0
 django-filter==2.2.0
 django-picklefield==2.1.1
+django-post-office==3.4.0
 django-select2==7.2.0
 django-settings-export==1.2.1
 djangorestframework==3.11.0
 dodgy==0.2.1
 isort==4.3.21
+jsonfield==3.1.0
 lazy-object-proxy==1.4.3
 mccabe==0.6.1
 mysqlclient==1.4.6

roles/webserver/templates/settings.j2

@@ -3,9 +3,14 @@
 
 SECRET_KEY={{ secret_key }}
 DEBUG={{ "True" if deploy_mode > 1 else "False" }}
-ALLOWED_HOSTS={{ inventory_hostname }},localhost,127.0.0.1
 DATABASE_URL=mysql://{{ db_user }}:{{ db_pass }}@localhost:3306/{{ db_name }}
 
+{% if allowed_hosts is defined %}
+ALLOWED_HOSTS={% for h in allowed_hosts %}{{ h }},{% endfor %}
+{% else %}
+ALLOWED_HOSTS={{ inventory_hostname }},localhost,127.0.0.1
+{% endif %}
+
 PROJECT_SHORT_NAME={{ display_short_name }}
 PROJECT_LONG_NAME={{ display_long_name }}