From 57c29bf01d4db1d327e6e8d78e92f3dfc80e6c2e Mon Sep 17 00:00:00 2001 From: James Graham Date: Thu, 16 Apr 2020 14:07:14 +0100 Subject: [PATCH 01/10] refactor: Begin import of customisation app --- breccia_mapper/settings.py | 25 +++++++++++++++++++++++++ breccia_mapper/views.py | 4 +++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/breccia_mapper/settings.py b/breccia_mapper/settings.py index 16d6569..29b27d3 100644 --- a/breccia_mapper/settings.py +++ b/breccia_mapper/settings.py @@ -14,6 +14,8 @@ https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ """ import collections +import logging +import logging.config import pathlib from django.urls import reverse_lazy @@ -241,6 +243,12 @@ LOGGING = { } } +# Initialise logger now so we can use it in this file + +LOGGING_CONFIG = None +logging.config.dictConfig(LOGGING) +logger = logging.getLogger(__name__) + # Admin panel variables @@ -262,3 +270,20 @@ CONSTANCE_BACKEND = 'constance.backends.database.DatabaseBackend' BOOTSTRAP4 = { 'include_jquery': 'full', } + + +# Import customisation app settings if present + +TEMPLATE_NAME_INDEX = 'index.html' + +try: + from custom.settings import ( + CUSTOMISATION_NAME, + TEMPLATE_NAME_INDEX + ) + logger.info("Loaded customisation app: %s", CUSTOMISATION_NAME) + + INSTALLED_APPS.append('custom') + +except ImportError as e: + logger.info("No customisation app loaded: %s", e) diff --git a/breccia_mapper/views.py b/breccia_mapper/views.py index 272f7c9..ff539f6 100644 --- a/breccia_mapper/views.py +++ b/breccia_mapper/views.py @@ -4,8 +4,10 @@ Views belonging to the core of the project. These views don't represent any of the models in the apps. """ +from django.conf import settings from django.views.generic import TemplateView class IndexView(TemplateView): - template_name = 'index.html' + # Template set in Django settings file - may be customised by a customisation app + template_name = settings.TEMPLATE_NAME_INDEX From fa07b13fbdfce3884161ba5fc90d7d1f3c656d72 Mon Sep 17 00:00:00 2001 From: James Graham Date: Thu, 16 Apr 2020 14:07:43 +0100 Subject: [PATCH 02/10] fix: Minor changes to style of masthead image --- breccia_mapper/static/css/masthead.css | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/breccia_mapper/static/css/masthead.css b/breccia_mapper/static/css/masthead.css index 56fb41e..79b9aa6 100644 --- a/breccia_mapper/static/css/masthead.css +++ b/breccia_mapper/static/css/masthead.css @@ -1,14 +1,14 @@ header.masthead { position: relative; background: #343a40 no-repeat center; - -webkit-background-size: cover; - -moz-background-size: cover; - -o-background-size: cover; - background-size: cover; + -webkit-background-size: contain; + -moz-background-size: contain; + -o-background-size: contain; + background-size: contain; padding-top: 8rem; padding-bottom: 8rem; - min-height: 200px; - height: 60vh; + min-height: 400px; + height: 40vh; z-index: -2; } From 2b0ba8d12ef7c5f4ed565b254f1fd45df84f9785 Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 17 Apr 2020 11:38:10 +0100 Subject: [PATCH 03/10] deploy: Add deployment keyfile variables to inventory --- .gitignore | 3 +-- roles/webserver/tasks/main.yml | 22 +++++++++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 848c6b0..206fcfe 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,7 @@ debug.log* # Configuration settings.ini -deployment-key -deployment-key.pub +deployment-key* # Deployment /.dbbackup/ diff --git a/roles/webserver/tasks/main.yml b/roles/webserver/tasks/main.yml index ef16b98..04103ab 100644 --- a/roles/webserver/tasks/main.yml +++ b/roles/webserver/tasks/main.yml @@ -63,20 +63,36 @@ - name: Copy deploy key copy: - src: 'deployment-key' + src: '{{ deployment_keyfile }}' dest: '/tmp/deployment-key' mode: 0600 - when: vagrant_dir.stat.exists == False + when: vagrant_dir.stat.exists == False and deployment_keyfile is defined - name: Clone / update from source repo git: repo: 'git@github.com:Southampton-RSG/breccia-mapper.git' dest: '{{ project_dir }}' - key_file: '/tmp/deployment-key' + key_file: '{{ "/tmp/deployment-key" if deployment_keyfile is defined else None }}' version: '{{ branch | default ("master") }}' accept_hostkey: yes when: vagrant_dir.stat.exists == False +- name: Copy customisation deploy key + copy: + src: '{{ customisation_repo_keyfile }}' + dest: '/tmp/deployment-key-customisation' + mode: 0600 + when: customisation_repo_keyfile is defined + +- name: Clone / update from customisation repo + git: + repo: '{{ customisation_repo }}' + dest: '{{ project_dir }}/custom' + key_file: '{{ "/tmp/deployment-key-customisation" if customisation_repo_keyfile is defined else None }}' + version: '{{ branch | default ("master") }}' + accept_hostkey: yes + when: customisation_repo is defined + - name: Copy and populate settings template template: src: 'settings.j2' From 19735e97719205679bf5ff2c1442237e19d183a1 Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 17 Apr 2020 11:38:59 +0100 Subject: [PATCH 04/10] fix: Fix network JS to use new serialized relationship format --- people/templates/people/network.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/people/templates/people/network.html b/people/templates/people/network.html index 144a27a..b55b2c7 100644 --- a/people/templates/people/network.html +++ b/people/templates/people/network.html @@ -104,8 +104,8 @@ group: 'edges', data: { id: 'relationship-' + relationship.pk.toString(), - source: 'person-' + relationship.source.toString(), - target: 'person-' + relationship.target.toString() + source: 'person-' + relationship.source.pk.toString(), + target: 'person-' + relationship.target.pk.toString() } }) } From 75fc169630a2c2fa82476f4336e19d1b6950841a Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 17 Apr 2020 15:04:48 +0100 Subject: [PATCH 05/10] fix: Fix filters on network view Filters now use OR for multiple choices in the same field --- people/forms.py | 9 +++------ people/views/network.py | 23 ++++++++++++++--------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/people/forms.py b/people/forms.py index d230fbb..1b242dc 100644 --- a/people/forms.py +++ b/people/forms.py @@ -35,7 +35,7 @@ class PersonForm(forms.ModelForm): class DynamicAnswerSetBase(forms.Form): - field_class = forms.ChoiceField + field_class = forms.ModelChoiceField field_widget = None field_required = True @@ -43,11 +43,8 @@ class DynamicAnswerSetBase(forms.Form): super().__init__(*args, **kwargs) for question in models.RelationshipQuestion.objects.all(): - # Get choices from model and add default 'not selected' option - choices = question.choices + [['', '---------']] - field = self.field_class(label=question, - choices=choices, + queryset=question.answers, widget=self.field_widget, required=self.field_required) self.fields['question_{}'.format(question.pk)] = field @@ -85,6 +82,6 @@ class NetworkFilterForm(DynamicAnswerSetBase): """ Form to provide filtering on the network view. """ - field_class = forms.MultipleChoiceField + field_class = forms.ModelMultipleChoiceField field_widget = Select2MultipleWidget field_required = False diff --git a/people/views/network.py b/people/views/network.py index 3e02efa..117557d 100644 --- a/people/views/network.py +++ b/people/views/network.py @@ -4,6 +4,7 @@ Views for displaying networks of :class:`People` and :class:`Relationship`s. from django.contrib.auth.mixins import LoginRequiredMixin from django.db.models import Q +from django.forms import ValidationError from django.utils import timezone from django.views.generic import FormView @@ -38,22 +39,22 @@ class NetworkView(LoginRequiredMixin, FormView): Add filtered QuerySets of :class:`Person` and :class:`Relationship` to the context. """ context = super().get_context_data(**kwargs) - form = context['form'] + form: forms.NetworkFilterForm = context['form'] + if not form.is_valid(): + raise ValidationError at_time = timezone.now() relationship_set = models.Relationship.objects.all() # Filter answers to relationship questions - for key, value in form.data.items(): - if key.startswith('question_') and value: - question_id = key.replace('question_', '', 1) - answer = models.RelationshipQuestionChoice.objects.get(pk=value, - question__pk=question_id) + for field, values in form.cleaned_data.items(): + if field.startswith('question_') and values: relationship_set = relationship_set.filter( + # Time filters must be here Q(answer_sets__replaced_timestamp__gt=at_time) | Q(answer_sets__replaced_timestamp__isnull=True), answer_sets__timestamp__lte=at_time, - answer_sets__question_answers=answer + answer_sets__question_answers__in=values ) context['person_set'] = serializers.PersonSerializer( @@ -62,11 +63,15 @@ class NetworkView(LoginRequiredMixin, FormView): ).data context['relationship_set'] = serializers.RelationshipSerializer( - relationship_set, + relationship_set.distinct(), many=True ).data return context def form_valid(self, form): - return self.render_to_response(self.get_context_data()) + try: + return self.render_to_response(self.get_context_data()) + + except ValidationError: + return self.form_invalid(form) From bf472a69fdfdb5182a83a6d64909e9662637be33 Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 17 Apr 2020 16:03:11 +0100 Subject: [PATCH 06/10] refactor: Reverse relationship query in network view --- people/views/network.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/people/views/network.py b/people/views/network.py index 117557d..970acc4 100644 --- a/people/views/network.py +++ b/people/views/network.py @@ -3,7 +3,7 @@ Views for displaying networks of :class:`People` and :class:`Relationship`s. """ from django.contrib.auth.mixins import LoginRequiredMixin -from django.db.models import Q +from django.db.models import F, Q from django.forms import ValidationError from django.utils import timezone from django.views.generic import FormView @@ -45,16 +45,16 @@ class NetworkView(LoginRequiredMixin, FormView): at_time = timezone.now() - relationship_set = models.Relationship.objects.all() + relationship_answerset_set = models.RelationshipAnswerSet.objects.filter( + Q(replaced_timestamp__gt=at_time) | Q(replaced_timestamp__isnull=True), + timestamp__lte=at_time + ) # Filter answers to relationship questions for field, values in form.cleaned_data.items(): if field.startswith('question_') and values: - relationship_set = relationship_set.filter( - # Time filters must be here - Q(answer_sets__replaced_timestamp__gt=at_time) | Q(answer_sets__replaced_timestamp__isnull=True), - answer_sets__timestamp__lte=at_time, - answer_sets__question_answers__in=values + relationship_answerset_set = relationship_answerset_set.filter( + question_answers__in=values ) context['person_set'] = serializers.PersonSerializer( @@ -63,7 +63,9 @@ class NetworkView(LoginRequiredMixin, FormView): ).data context['relationship_set'] = serializers.RelationshipSerializer( - relationship_set.distinct(), + models.Relationship.objects.filter( + pk__in=relationship_answerset_set.values_list('relationship', flat=True) + ), many=True ).data From 719b11e79e43bff369981a5c121b6f6258dac658 Mon Sep 17 00:00:00 2001 From: James Graham Date: Mon, 20 Apr 2020 13:30:15 +0100 Subject: [PATCH 07/10] fix: Fix broken relationship update form Did not get values from fields correctly Incorrectly marked answersets as expired immediately --- people/forms.py | 7 ++----- people/views/relationship.py | 7 +++---- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/people/forms.py b/people/forms.py index 1b242dc..7757847 100644 --- a/people/forms.py +++ b/people/forms.py @@ -69,11 +69,8 @@ class RelationshipAnswerSetForm(forms.ModelForm, DynamicAnswerSetBase): if commit: # Save answers to relationship questions for key, value in self.cleaned_data.items(): - if key.startswith('question_'): - question_id = key.replace('question_', '', 1) - answer = models.RelationshipQuestionChoice.objects.get(pk=value, - question__pk=question_id) - self.instance.question_answers.add(answer) + if key.startswith('question_') and value: + self.instance.question_answers.add(value) return self.instance diff --git a/people/views/relationship.py b/people/views/relationship.py index eeb2ea6..8709b99 100644 --- a/people/views/relationship.py +++ b/people/views/relationship.py @@ -118,13 +118,12 @@ class RelationshipUpdateView(permissions.UserIsLinkedPersonMixin, CreateView): """ Mark any previous answer sets as replaced. """ - previous_valid_answer_sets = self.relationship.answer_sets.filter(replaced_timestamp__isnull=True) - response = super().form_valid(form) + now_date = timezone.now().date() # Shouldn't be more than one after initial updates after migration - for answer_set in previous_valid_answer_sets: - answer_set.replaced_timestamp = timezone.now() + for answer_set in self.relationship.answer_sets.exclude(pk=self.object.pk): + answer_set.replaced_timestamp = now_date answer_set.save() return response From 3b3cec02becd556d7407c8e2ff24eae190dcdd3f Mon Sep 17 00:00:00 2001 From: James Graham Date: Mon, 20 Apr 2020 13:32:07 +0100 Subject: [PATCH 08/10] feat: Add date field to network view Filters relationships to only those valid at date --- people/forms.py | 6 ++++++ people/views/network.py | 12 +++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/people/forms.py b/people/forms.py index 7757847..5877ce5 100644 --- a/people/forms.py +++ b/people/forms.py @@ -82,3 +82,9 @@ class NetworkFilterForm(DynamicAnswerSetBase): field_class = forms.ModelMultipleChoiceField field_widget = Select2MultipleWidget field_required = False + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Add date field to select relationships at a particular point in time + self.fields['date'] = forms.DateField(required=False) diff --git a/people/views/network.py b/people/views/network.py index 970acc4..a17ab9e 100644 --- a/people/views/network.py +++ b/people/views/network.py @@ -3,7 +3,7 @@ Views for displaying networks of :class:`People` and :class:`Relationship`s. """ from django.contrib.auth.mixins import LoginRequiredMixin -from django.db.models import F, Q +from django.db.models import Q from django.forms import ValidationError from django.utils import timezone from django.views.generic import FormView @@ -41,13 +41,15 @@ class NetworkView(LoginRequiredMixin, FormView): context = super().get_context_data(**kwargs) form: forms.NetworkFilterForm = context['form'] if not form.is_valid(): - raise ValidationError + return context - at_time = timezone.now() + at_date = form.cleaned_data['date'] + if not at_date: + at_date = timezone.now().date() relationship_answerset_set = models.RelationshipAnswerSet.objects.filter( - Q(replaced_timestamp__gt=at_time) | Q(replaced_timestamp__isnull=True), - timestamp__lte=at_time + Q(replaced_timestamp__date__gte=at_date) | Q(replaced_timestamp__isnull=True), + timestamp__date__lte=at_date ) # Filter answers to relationship questions From d6763a760e2a256df2c974de135cfb7f84b99934 Mon Sep 17 00:00:00 2001 From: James Graham Date: Mon, 20 Apr 2020 13:56:06 +0100 Subject: [PATCH 09/10] fix: Separate date field from relationship questions --- people/templates/people/network.html | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/people/templates/people/network.html b/people/templates/people/network.html index b55b2c7..0901d13 100644 --- a/people/templates/people/network.html +++ b/people/templates/people/network.html @@ -18,8 +18,12 @@

Filter Relationships

- {% load bootstrap4 %} - {% bootstrap_form form %} + {% load bootstrap4 %} + {% bootstrap_form form exclude='date' %} + +
+ {% bootstrap_field form.date %} +
@@ -27,12 +31,14 @@
+ {% buttons %} {% endbuttons %}
From 5dcfbb6052c5302588119a394551347949e1b133 Mon Sep 17 00:00:00 2001 From: James Graham Date: Mon, 20 Apr 2020 15:12:34 +0100 Subject: [PATCH 10/10] feat!: Add answer sets to CSV export BREAKING CHANGE: Change format of Relationship CSV export BREAKING CHANGE: Use 'id' for id field in CSV exports Add RelationshipAnswerSet CSV export --- export/serializers/people.py | 25 +++++++++++++++++++------ export/templates/export/export.html | 9 +++++++++ export/urls.py | 4 ++++ export/views/people.py | 5 +++++ 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/export/serializers/people.py b/export/serializers/people.py index 8c84ccb..196f295 100644 --- a/export/serializers/people.py +++ b/export/serializers/people.py @@ -11,7 +11,7 @@ class SimplePersonSerializer(serializers.ModelSerializer): class Meta: model = models.Person fields = [ - 'pk', + 'id', 'name', ] @@ -20,7 +20,7 @@ class PersonSerializer(base.FlattenedModelSerializer): class Meta: model = models.Person fields = [ - 'pk', + 'id', 'name', 'core_member', 'gender', @@ -28,8 +28,8 @@ class PersonSerializer(base.FlattenedModelSerializer): 'nationality', 'country_of_residence', ] - - + + class RelationshipSerializer(base.FlattenedModelSerializer): source = SimplePersonSerializer() target = SimplePersonSerializer() @@ -37,11 +37,24 @@ class RelationshipSerializer(base.FlattenedModelSerializer): class Meta: model = models.Relationship fields = [ - 'pk', + 'id', 'source', 'target', ] + +class RelationshipAnswerSetSerializer(base.FlattenedModelSerializer): + relationship = RelationshipSerializer() + + class Meta: + model = models.RelationshipAnswerSet + fields = [ + 'id', + 'relationship', + 'timestamp', + 'replaced_timestamp', + ] + @property def column_headers(self) -> typing.List[str]: headers = super().column_headers @@ -57,7 +70,7 @@ class RelationshipSerializer(base.FlattenedModelSerializer): try: # Add relationship question answers to data - for answer in instance.current_answers.question_answers.all(): + for answer in instance.question_answers.all(): rep[answer.question.slug.replace('-', '_')] = answer.slug.replace('-', '_') except AttributeError: diff --git a/export/templates/export/export.html b/export/templates/export/export.html index c94b2df..28a04f6 100644 --- a/export/templates/export/export.html +++ b/export/templates/export/export.html @@ -39,6 +39,15 @@ + + Relationship Answer Sets + + + Export + + + Activities diff --git a/export/urls.py b/export/urls.py index c56fca5..8fed824 100644 --- a/export/urls.py +++ b/export/urls.py @@ -17,6 +17,10 @@ urlpatterns = [ path('export/relationships', views.people.RelationshipExportView.as_view(), name='relationship'), + + path('export/relationship-answer-sets', + views.people.RelationshipAnswerSetExportView.as_view(), + name='relationship-answer-set'), path('export/activities', views.activities.ActivityExportView.as_view(), diff --git a/export/views/people.py b/export/views/people.py index 7f73d5f..a9ff4a6 100644 --- a/export/views/people.py +++ b/export/views/people.py @@ -12,3 +12,8 @@ class PersonExportView(base.CsvExportView): class RelationshipExportView(base.CsvExportView): model = models.relationship.Relationship serializer_class = serializers.people.RelationshipSerializer + + +class RelationshipAnswerSetExportView(base.CsvExportView): + model = models.relationship.RelationshipAnswerSet + serializer_class = serializers.people.RelationshipAnswerSetSerializer