From 9f493a53e4c842f8b03e175690188c848cbaf2a2 Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 27 Mar 2020 11:15:26 +0000 Subject: [PATCH 01/18] feat: Add initial CSV export of people --- breccia_mapper/settings.py | 9 +++++++++ requirements.txt | 2 ++ 2 files changed, 11 insertions(+) diff --git a/breccia_mapper/settings.py b/breccia_mapper/settings.py index 5706267..22aa5ee 100644 --- a/breccia_mapper/settings.py +++ b/breccia_mapper/settings.py @@ -246,3 +246,12 @@ CONSTANCE_BACKEND = 'constance.backends.database.DatabaseBackend' BOOTSTRAP4 = { 'include_jquery': 'full', } + + +# Django Rest Framework settings for API + +REST_FRAMEWORK = { + 'DEFAULT_RENDERER_CLASSES': [ + 'rest_framework_csv.renderers.CSVRenderer', + ] +} diff --git a/requirements.txt b/requirements.txt index 1c63c7e..fb959e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ django-picklefield==2.1.1 django-select2==7.2.0 django-settings-export==1.2.1 djangorestframework==3.11.0 +djangorestframework-csv==2.1.0 dodgy==0.2.1 isort==4.3.21 lazy-object-proxy==1.4.3 @@ -38,4 +39,5 @@ snowballstemmer==2.0.0 soupsieve==1.9.5 sqlparse==0.3.0 typed-ast==1.4.1 +unicodecsv==0.14.1 wrapt==1.11.2 From af77cb39f8dac868a7f420a88aad69b49ff5ef94 Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 27 Mar 2020 14:31:38 +0000 Subject: [PATCH 02/18] feat: Add CSV export of Relationships --- breccia_mapper/settings.py | 14 +++++--------- people/serializers.py | 19 ++++++++++++++++++- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/breccia_mapper/settings.py b/breccia_mapper/settings.py index 20a0aa8..795ae33 100644 --- a/breccia_mapper/settings.py +++ b/breccia_mapper/settings.py @@ -141,6 +141,11 @@ REST_FRAMEWORK = { 'DEFAULT_PERMISSION_CLASSES': [ 'rest_framework.permissions.IsAuthenticated', ], + 'DEFAULT_RENDERER_CLASSES': [ + 'rest_framework.renderers.JSONRenderer', + 'rest_framework_csv.renderers.CSVRenderer', + 'rest_framework.renderers.BrowsableAPIRenderer', + ], } @@ -256,12 +261,3 @@ CONSTANCE_BACKEND = 'constance.backends.database.DatabaseBackend' BOOTSTRAP4 = { 'include_jquery': 'full', } - - -# Django Rest Framework settings for API - -REST_FRAMEWORK = { - 'DEFAULT_RENDERER_CLASSES': [ - 'rest_framework_csv.renderers.CSVRenderer', - ] -} diff --git a/people/serializers.py b/people/serializers.py index 3c98db3..b3f6c4f 100644 --- a/people/serializers.py +++ b/people/serializers.py @@ -14,9 +14,26 @@ class PersonSerializer(serializers.ModelSerializer): 'pk', 'name', ] - + + +class MinimalPersonSerializer(serializers.ModelSerializer): + """ + Serializer containing just the necessary fields to identify a :class:`Person`. + + Used for nesting within other serializers. + """ + class Meta: + model = models.Person + fields = [ + 'pk', + 'name', + ] + class RelationshipSerializer(serializers.ModelSerializer): + source = MinimalPersonSerializer() + target = MinimalPersonSerializer() + class Meta: model = models.Relationship fields = [ From 74aab162e10bcac43ee6933601838ddc1a2ac3ad Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 27 Mar 2020 14:31:59 +0000 Subject: [PATCH 03/18] feat: Add export front page --- breccia_mapper/templates/export.html | 43 ++++++++++++++++++++++++++++ breccia_mapper/urls.py | 4 +++ breccia_mapper/views.py | 4 +++ 3 files changed, 51 insertions(+) create mode 100644 breccia_mapper/templates/export.html diff --git a/breccia_mapper/templates/export.html b/breccia_mapper/templates/export.html new file mode 100644 index 0000000..c552ced --- /dev/null +++ b/breccia_mapper/templates/export.html @@ -0,0 +1,43 @@ +{% extends 'base.html' %} + +{% block content %} + + +
+ + + + + + + + + + + + + + + + + + + + + + + +
TypeRecords
People + Export +
Relationships + Export +
+ + +{% endblock %} diff --git a/breccia_mapper/urls.py b/breccia_mapper/urls.py index e5f431b..b8ed941 100644 --- a/breccia_mapper/urls.py +++ b/breccia_mapper/urls.py @@ -28,6 +28,10 @@ urlpatterns = [ views.IndexView.as_view(), name='index'), + path('export', + views.ExportListView.as_view(), + name='export'), + path('', include('people.urls')), diff --git a/breccia_mapper/views.py b/breccia_mapper/views.py index 2844b2c..33ed8cd 100644 --- a/breccia_mapper/views.py +++ b/breccia_mapper/views.py @@ -3,3 +3,7 @@ from django.views.generic import TemplateView class IndexView(TemplateView): template_name = 'index.html' + + +class ExportListView(TemplateView): + template_name = 'export.html' From 7c75f9d7f43bdb8b7c27320d7d1acaadb806d515 Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 27 Mar 2020 14:33:22 +0000 Subject: [PATCH 04/18] feat: Add export page link to navbar --- breccia_mapper/templates/base.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/breccia_mapper/templates/base.html b/breccia_mapper/templates/base.html index 123d2ea..ef6c828 100644 --- a/breccia_mapper/templates/base.html +++ b/breccia_mapper/templates/base.html @@ -78,6 +78,10 @@ {% if request.user.is_superuser %} +
  • + Export +
    • +
  • Admin
    • From 2cdc7675c70afd0e7a4b2721dc465f0344ac55a3 Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 27 Mar 2020 15:15:18 +0000 Subject: [PATCH 05/18] refactor: Split people.views into multiple modules --- people/urls.py | 22 ++-- people/views/__init__.py | 9 ++ people/views/network.py | 72 ++++++++++ people/views/person.py | 78 +++++++++++ people/{views.py => views/relationship.py} | 146 ++------------------- 5 files changed, 178 insertions(+), 149 deletions(-) create mode 100644 people/views/__init__.py create mode 100644 people/views/network.py create mode 100644 people/views/person.py rename people/{views.py => views/relationship.py} (50%) diff --git a/people/urls.py b/people/urls.py index de38945..d484424 100644 --- a/people/urls.py +++ b/people/urls.py @@ -7,46 +7,46 @@ app_name = 'people' urlpatterns = [ path('profile/', - views.ProfileView.as_view(), + views.person.ProfileView.as_view(), name='person.profile'), path('people/create', - views.PersonCreateView.as_view(), + views.person.PersonCreateView.as_view(), name='person.create'), path('people', - views.PersonListView.as_view(), + views.person.PersonListView.as_view(), name='person.list'), path('people/', - views.ProfileView.as_view(), + views.person.ProfileView.as_view(), name='person.detail'), path('people//update', - views.PersonUpdateView.as_view(), + views.person.PersonUpdateView.as_view(), name='person.update'), path('people//relationships/create', - views.RelationshipCreateView.as_view(), + views.relationship.RelationshipCreateView.as_view(), name='person.relationship.create'), path('relationships/', - views.RelationshipDetailView.as_view(), + views.relationship.RelationshipDetailView.as_view(), name='relationship.detail'), path('relationships//update', - views.RelationshipUpdateView.as_view(), + views.relationship.RelationshipUpdateView.as_view(), name='relationship.update'), path('api/people', - views.PersonApiView.as_view(), + views.person.PersonApiView.as_view(), name='person.api.list'), path('api/relationships', - views.RelationshipApiView.as_view(), + views.relationship.RelationshipApiView.as_view(), name='relationship.api.list'), path('network', - views.NetworkView.as_view(), + views.network.NetworkView.as_view(), name='network'), ] diff --git a/people/views/__init__.py b/people/views/__init__.py new file mode 100644 index 0000000..39dd5c9 --- /dev/null +++ b/people/views/__init__.py @@ -0,0 +1,9 @@ +""" +Views for displaying or manipulating models within the `people` app. +""" + +from . import ( + network, + person, + relationship +) diff --git a/people/views/network.py b/people/views/network.py new file mode 100644 index 0000000..3e02efa --- /dev/null +++ b/people/views/network.py @@ -0,0 +1,72 @@ +""" +Views for displaying networks of :class:`People` and :class:`Relationship`s. +""" + +from django.contrib.auth.mixins import LoginRequiredMixin +from django.db.models import Q +from django.utils import timezone +from django.views.generic import FormView + + +from people import forms, models, serializers + + +class NetworkView(LoginRequiredMixin, FormView): + """ + View to display relationship network. + """ + template_name = 'people/network.html' + form_class = forms.NetworkFilterForm + + def get_form_kwargs(self): + """ + Add GET params to form data. + """ + kwargs = super().get_form_kwargs() + + if self.request.method == 'GET': + if 'data' in kwargs: + kwargs['data'].update(self.request.GET) + + else: + kwargs['data'] = self.request.GET + + return kwargs + + def get_context_data(self, **kwargs): + """ + Add filtered QuerySets of :class:`Person` and :class:`Relationship` to the context. + """ + context = super().get_context_data(**kwargs) + form = context['form'] + + at_time = timezone.now() + + relationship_set = models.Relationship.objects.all() + + # Filter answers to relationship questions + for key, value in form.data.items(): + if key.startswith('question_') and value: + question_id = key.replace('question_', '', 1) + answer = models.RelationshipQuestionChoice.objects.get(pk=value, + question__pk=question_id) + relationship_set = relationship_set.filter( + Q(answer_sets__replaced_timestamp__gt=at_time) | Q(answer_sets__replaced_timestamp__isnull=True), + answer_sets__timestamp__lte=at_time, + answer_sets__question_answers=answer + ) + + context['person_set'] = serializers.PersonSerializer( + models.Person.objects.all(), + many=True + ).data + + context['relationship_set'] = serializers.RelationshipSerializer( + relationship_set, + many=True + ).data + + return context + + def form_valid(self, form): + return self.render_to_response(self.get_context_data()) diff --git a/people/views/person.py b/people/views/person.py new file mode 100644 index 0000000..c868e9d --- /dev/null +++ b/people/views/person.py @@ -0,0 +1,78 @@ +""" +Views for displaying or manipulating instances of :class:`Person`. +""" + +from django.views.generic import CreateView, DetailView, ListView, UpdateView + +from rest_framework.views import APIView, Response + +from people import forms, models, permissions, serializers + + +class PersonCreateView(CreateView): + """ + View to create a new instance of :class:`Person`. + + If 'user' is passed as a URL parameter - link the new person to the current user. + """ + model = models.Person + template_name = 'people/person/create.html' + form_class = forms.PersonForm + + def form_valid(self, form): + if 'user' in self.request.GET: + form.instance.user = self.request.user + + return super().form_valid(form) + + +class PersonListView(ListView): + """ + View displaying a list of :class:`Person` objects - searchable. + """ + model = models.Person + template_name = 'people/person/list.html' + + +class ProfileView(permissions.UserIsLinkedPersonMixin, DetailView): + """ + View displaying the profile of a :class:`Person` - who may be a user. + """ + model = models.Person + template_name = 'people/person/detail.html' + + def get_object(self, queryset=None) -> models.Person: + """ + Get the :class:`Person` object to be represented by this page. + + If not determined from url get current user. + """ + try: + return super().get_object(queryset) + + except AttributeError: + # pk was not provided in URL + return self.request.user.person + + +class PersonUpdateView(permissions.UserIsLinkedPersonMixin, UpdateView): + """ + View for updating a :class:`Person` record. + """ + model = models.Person + template_name = 'people/person/update.html' + form_class = forms.PersonForm + + +class PersonApiView(APIView): + """ + List all :class:`Person` instances. + """ + + def get(self, request, format=None): + """ + List all :class:`Person` instances. + """ + serializer = serializers.PersonSerializer(models.Person.objects.all(), + many=True) + return Response(serializer.data) diff --git a/people/views.py b/people/views/relationship.py similarity index 50% rename from people/views.py rename to people/views/relationship.py index 35c06ea..3e2c7d5 100644 --- a/people/views.py +++ b/people/views/relationship.py @@ -1,71 +1,14 @@ """ -Views for displaying or manipulating models in the 'people' app. +Views for displaying or manipulating instances of :class:`Relationship`. """ -from django.contrib.auth.mixins import LoginRequiredMixin -from django.db.models import Q from django.urls import reverse from django.utils import timezone -from django.views.generic import CreateView, DetailView, FormView, ListView, UpdateView +from django.views.generic import CreateView, DetailView from rest_framework.views import APIView, Response -from . import forms, models, permissions, serializers - - -class PersonCreateView(CreateView): - """ - View to create a new instance of :class:`Person`. - - If 'user' is passed as a URL parameter - link the new person to the current user. - """ - model = models.Person - template_name = 'people/person/create.html' - form_class = forms.PersonForm - - def form_valid(self, form): - if 'user' in self.request.GET: - form.instance.user = self.request.user - - return super().form_valid(form) - - -class PersonListView(ListView): - """ - View displaying a list of :class:`Person` objects - searchable. - """ - model = models.Person - template_name = 'people/person/list.html' - - -class ProfileView(permissions.UserIsLinkedPersonMixin, DetailView): - """ - View displaying the profile of a :class:`Person` - who may be a user. - """ - model = models.Person - template_name = 'people/person/detail.html' - - def get_object(self, queryset=None) -> models.Person: - """ - Get the :class:`Person` object to be represented by this page. - - If not determined from url get current user. - """ - try: - return super().get_object(queryset) - - except AttributeError: - # pk was not provided in URL - return self.request.user.person - - -class PersonUpdateView(permissions.UserIsLinkedPersonMixin, UpdateView): - """ - View for updating a :class:`Person` record. - """ - model = models.Person - template_name = 'people/person/update.html' - form_class = forms.PersonForm +from people import forms, models, permissions, serializers class RelationshipDetailView(permissions.UserIsLinkedPersonMixin, DetailView): @@ -165,12 +108,12 @@ class RelationshipUpdateView(permissions.UserIsLinkedPersonMixin, CreateView): context['relationship'] = self.relationship return context - + def get_initial(self): initial = super().get_initial() - + initial['relationship'] = self.relationship - + return initial def form_valid(self, form): @@ -187,25 +130,13 @@ class RelationshipUpdateView(permissions.UserIsLinkedPersonMixin, CreateView): answer_set.save() return response - -class PersonApiView(APIView): - """ - List all :class:`Person` instances. - """ - def get(self, request, format=None): - """ - List all :class:`Person` instances. - """ - serializer = serializers.PersonSerializer(models.Person.objects.all(), - many=True) - return Response(serializer.data) - - + class RelationshipApiView(APIView): """ List all :class:`Relationship` instances. """ + def get(self, request, format=None): """ List all :class:`Relationship` instances. @@ -213,64 +144,3 @@ class RelationshipApiView(APIView): serializer = serializers.RelationshipSerializer(models.Relationship.objects.all(), many=True) return Response(serializer.data) - - -class NetworkView(LoginRequiredMixin, FormView): - """ - View to display relationship network. - """ - template_name = 'people/network.html' - form_class = forms.NetworkFilterForm - - def get_form_kwargs(self): - """ - Add GET params to form data. - """ - kwargs = super().get_form_kwargs() - - if self.request.method == 'GET': - if 'data' in kwargs: - kwargs['data'].update(self.request.GET) - - else: - kwargs['data'] = self.request.GET - - return kwargs - - def get_context_data(self, **kwargs): - """ - Add filtered QuerySets of :class:`Person` and :class:`Relationship` to the context. - """ - context = super().get_context_data(**kwargs) - form = context['form'] - - at_time = timezone.now() - - relationship_set = models.Relationship.objects.all() - - # Filter answers to relationship questions - for key, value in form.data.items(): - if key.startswith('question_') and value: - question_id = key.replace('question_', '', 1) - answer = models.RelationshipQuestionChoice.objects.get(pk=value, - question__pk=question_id) - relationship_set = relationship_set.filter( - Q(answer_sets__replaced_timestamp__gt=at_time) | Q(answer_sets__replaced_timestamp__isnull=True), - answer_sets__timestamp__lte=at_time, - answer_sets__question_answers=answer - ) - - context['person_set'] = serializers.PersonSerializer( - models.Person.objects.all(), - many=True - ).data - - context['relationship_set'] = serializers.RelationshipSerializer( - relationship_set, - many=True - ).data - - return context - - def form_valid(self, form): - return self.render_to_response(self.get_context_data()) From da57108e3e8cf4ff03cbd7b79b62d69d293ab441 Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 27 Mar 2020 17:00:37 +0000 Subject: [PATCH 06/18] refactor: Use custom CSV export view Remove dependency on djangorestframework-csv --- breccia_mapper/templates/export.html | 4 ++-- people/serializers.py | 15 +++++-------- people/urls.py | 12 +++++----- people/views/__init__.py | 1 + people/views/export.py | 33 ++++++++++++++++++++++++++++ people/views/person.py | 18 +-------------- people/views/relationship.py | 18 +-------------- requirements.txt | 2 -- 8 files changed, 50 insertions(+), 53 deletions(-) create mode 100644 people/views/export.py diff --git a/breccia_mapper/templates/export.html b/breccia_mapper/templates/export.html index c552ced..e0aca40 100644 --- a/breccia_mapper/templates/export.html +++ b/breccia_mapper/templates/export.html @@ -24,7 +24,7 @@ Export + href="{% url 'people:person.export' %}">Export @@ -33,7 +33,7 @@ Export + href="{% url 'people:relationship.export' %}">Export diff --git a/people/serializers.py b/people/serializers.py index b3f6c4f..85e1c27 100644 --- a/people/serializers.py +++ b/people/serializers.py @@ -16,24 +16,21 @@ class PersonSerializer(serializers.ModelSerializer): ] -class MinimalPersonSerializer(serializers.ModelSerializer): - """ - Serializer containing just the necessary fields to identify a :class:`Person`. - - Used for nesting within other serializers. - """ +class PersonExportSerializer(serializers.ModelSerializer): class Meta: model = models.Person fields = [ 'pk', 'name', + 'core_member', + 'gender', + 'age_group', + 'nationality', + 'country_of_residence', ] class RelationshipSerializer(serializers.ModelSerializer): - source = MinimalPersonSerializer() - target = MinimalPersonSerializer() - class Meta: model = models.Relationship fields = [ diff --git a/people/urls.py b/people/urls.py index d484424..784522c 100644 --- a/people/urls.py +++ b/people/urls.py @@ -38,13 +38,13 @@ urlpatterns = [ views.relationship.RelationshipUpdateView.as_view(), name='relationship.update'), - path('api/people', - views.person.PersonApiView.as_view(), - name='person.api.list'), + path('people/export', + views.export.PersonExportView.as_view(), + name='person.export'), - path('api/relationships', - views.relationship.RelationshipApiView.as_view(), - name='relationship.api.list'), + path('relationships/export', + views.export.RelationshipExportView.as_view(), + name='relationship.export'), path('network', views.network.NetworkView.as_view(), diff --git a/people/views/__init__.py b/people/views/__init__.py index 39dd5c9..5dd2c46 100644 --- a/people/views/__init__.py +++ b/people/views/__init__.py @@ -3,6 +3,7 @@ Views for displaying or manipulating models within the `people` app. """ from . import ( + export, network, person, relationship diff --git a/people/views/export.py b/people/views/export.py new file mode 100644 index 0000000..343b7f7 --- /dev/null +++ b/people/views/export.py @@ -0,0 +1,33 @@ +import csv +import typing + +from django.http import HttpResponse +from django.views.generic.list import BaseListView + +from .. import models, serializers + + +class CsvExportView(BaseListView): + model = None + serializer_class = None + + def render_to_response(self, context: typing.Dict) -> HttpResponse: + response = HttpResponse(content_type='text/csv') + response['Content-Disposition'] = f'attachment; filename="{self.get_context_object_name(self.object_list)}.csv"' + + serializer = self.serializer_class(self.get_queryset(), many=True) + writer = csv.DictWriter(response, fieldnames=self.serializer_class.Meta.fields) + writer.writeheader() + writer.writerows(serializer.data) + + return response + + +class PersonExportView(CsvExportView): + model = models.Person + serializer_class = serializers.PersonExportSerializer + + +class RelationshipExportView(CsvExportView): + model = models.Relationship + serializer_class = serializers.RelationshipSerializer diff --git a/people/views/person.py b/people/views/person.py index c868e9d..1ba28f5 100644 --- a/people/views/person.py +++ b/people/views/person.py @@ -4,9 +4,7 @@ Views for displaying or manipulating instances of :class:`Person`. from django.views.generic import CreateView, DetailView, ListView, UpdateView -from rest_framework.views import APIView, Response - -from people import forms, models, permissions, serializers +from people import forms, models, permissions class PersonCreateView(CreateView): @@ -62,17 +60,3 @@ class PersonUpdateView(permissions.UserIsLinkedPersonMixin, UpdateView): model = models.Person template_name = 'people/person/update.html' form_class = forms.PersonForm - - -class PersonApiView(APIView): - """ - List all :class:`Person` instances. - """ - - def get(self, request, format=None): - """ - List all :class:`Person` instances. - """ - serializer = serializers.PersonSerializer(models.Person.objects.all(), - many=True) - return Response(serializer.data) diff --git a/people/views/relationship.py b/people/views/relationship.py index 3e2c7d5..eeb2ea6 100644 --- a/people/views/relationship.py +++ b/people/views/relationship.py @@ -6,9 +6,7 @@ from django.urls import reverse from django.utils import timezone from django.views.generic import CreateView, DetailView -from rest_framework.views import APIView, Response - -from people import forms, models, permissions, serializers +from people import forms, models, permissions class RelationshipDetailView(permissions.UserIsLinkedPersonMixin, DetailView): @@ -130,17 +128,3 @@ class RelationshipUpdateView(permissions.UserIsLinkedPersonMixin, CreateView): answer_set.save() return response - - -class RelationshipApiView(APIView): - """ - List all :class:`Relationship` instances. - """ - - def get(self, request, format=None): - """ - List all :class:`Relationship` instances. - """ - serializer = serializers.RelationshipSerializer(models.Relationship.objects.all(), - many=True) - return Response(serializer.data) diff --git a/requirements.txt b/requirements.txt index fb959e2..1c63c7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,6 @@ django-picklefield==2.1.1 django-select2==7.2.0 django-settings-export==1.2.1 djangorestframework==3.11.0 -djangorestframework-csv==2.1.0 dodgy==0.2.1 isort==4.3.21 lazy-object-proxy==1.4.3 @@ -39,5 +38,4 @@ snowballstemmer==2.0.0 soupsieve==1.9.5 sqlparse==0.3.0 typed-ast==1.4.1 -unicodecsv==0.14.1 wrapt==1.11.2 From 74d3c1b091ea4e3059d930bdae264749981f8e3d Mon Sep 17 00:00:00 2001 From: James Graham Date: Mon, 30 Mar 2020 17:18:19 +0100 Subject: [PATCH 07/18] deploy: Use RedHat Software Collections RHSCL provides patched versions of Python and Nginx --- .gitignore | 3 +- Makefile | 4 +-- roles/database/tasks/main.yml | 1 + roles/webserver/defaults/main.yml | 2 +- roles/webserver/tasks/main.yml | 40 ++++++++++++++-------- roles/webserver/templates/uwsgi-service.j2 | 4 +-- 6 files changed, 34 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 5a7cc0c..0d631fd 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ deployment-key deployment-key.pub # Deployment +/.dbbackup/ .vagrant/ staging.yml -/.dbbackup/ +production.yml diff --git a/Makefile b/Makefile index adc709c..4187306 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,8 @@ lint: .PHONY: staging staging: - ansible-playbook -v -i staging.yml playbook.yml -u jag1e17 -K + env ANSIBLE_STDOUT_CALLBACK=debug ansible-playbook -v -i staging.yml playbook.yml -u jag1e17 -K .PHONY: production production: - ansible-playbook -v -i production.yml playbook.yml -u jag1e17 -K + env ANSIBLE_STDOUT_CALLBACK=debug ansible-playbook -v -i production.yml playbook.yml -u jag1e17 -K diff --git a/roles/database/tasks/main.yml b/roles/database/tasks/main.yml index 24c5fff..6bc72b9 100644 --- a/roles/database/tasks/main.yml +++ b/roles/database/tasks/main.yml @@ -22,6 +22,7 @@ name: mariadb state: restarted enabled: yes + daemon_reload: yes - name: Create database mysql_db: diff --git a/roles/webserver/defaults/main.yml b/roles/webserver/defaults/main.yml index cfb895e..5d9fbcf 100644 --- a/roles/webserver/defaults/main.yml +++ b/roles/webserver/defaults/main.yml @@ -5,7 +5,7 @@ deploy_mode_dict: 3: Development deploy_mode: 3 -secret_key: '{{ lookup("password", "/tmp/secretkeyfile") }}' +secret_key: '{{ lookup("password", "/dev/null") }}' project_name: 'breccia-mapper' project_full_name: 'breccia_mapper' diff --git a/roles/webserver/tasks/main.yml b/roles/webserver/tasks/main.yml index 74c6ab5..cc1504c 100644 --- a/roles/webserver/tasks/main.yml +++ b/roles/webserver/tasks/main.yml @@ -12,6 +12,17 @@ name: '*' state: latest +- name: Enable RedHat Software Collections - RHEL + rhsm_repository: + name: rhel-server-rhscl-7-rpms + when: ansible_distribution == "RedHat" + +- name: Enable RedHat Software Collections - CentOS + yum: + name: centos-release-scl + state: latest + when: ansible_distribution == "CentOS" + - name: Install system prerequisites yum: name: '{{ packages }}' @@ -20,12 +31,8 @@ packages: - gcc - git - - nginx - - python36 - - python36-devel - - python36-pip - - python36-setuptools - - python36-virtualenv + - rh-nginx114 + - rh-python36 - policycoreutils-python - python - python-setuptools @@ -86,11 +93,15 @@ group: '{{ web_group }}' recurse: yes +- name: Create venv + shell: | + source scl_source enable rh-python36 + python3 -m venv {{ venv_dir }} + - name: Install pip requirements pip: requirements: '{{ project_dir }}/requirements.txt' virtualenv: '{{ venv_dir }}' - virtualenv_command: virtualenv-3 - name: Create static directory file: @@ -124,10 +135,9 @@ when: deploy_mode > 1 - name: Install uWSGI - pip: - name: uwsgi - state: latest - executable: pip3 + shell: | + source scl_source enable rh-python36 + pip3 install uwsgi - name: Setup uWSGI config file: @@ -145,6 +155,7 @@ name: uwsgi state: started enabled: yes + daemon_reload: yes - name: Copy web config files template: @@ -189,7 +200,7 @@ - name: Copy Nginx site template: src: nginx-site-ssl.j2 - dest: '/etc/nginx/conf.d/{{ project_name }}-ssl.conf' + dest: '/etc/opt/rh/rh-nginx114/nginx/conf.d/{{ project_name }}-ssl.conf' owner: '{{ web_user }}' group: '{{ web_group }}' @@ -198,7 +209,7 @@ - name: Copy Nginx site template: src: nginx-site.j2 - dest: '/etc/nginx/conf.d/{{ project_name }}.conf' + dest: '/etc/opt/rh/rh-nginx114/nginx/conf.d/{{ project_name }}.conf' owner: '{{ web_user }}' group: '{{ web_group }}' @@ -207,9 +218,10 @@ name: "{{ item }}" state: restarted enabled: yes + daemon_reload: yes with_items: - uwsgi - - nginx + - rh-nginx114-nginx - name: Open webserver ports on firewall firewalld: diff --git a/roles/webserver/templates/uwsgi-service.j2 b/roles/webserver/templates/uwsgi-service.j2 index fffb99c..97f5738 100644 --- a/roles/webserver/templates/uwsgi-service.j2 +++ b/roles/webserver/templates/uwsgi-service.j2 @@ -2,8 +2,8 @@ Description=uWSGI Emperor Service [Service] -ExecStartPre=/bin/bash -c 'mkdir -p /run/uwsgi; chown {{ web_user }}:{{ web_group }} /run/uwsgi' -ExecStart=/usr/local/bin/uwsgi --emperor /etc/uwsgi/sites +ExecStartPre=/bin/bash -c 'mkdir -p /run/uwsgi; chown {{ web_user }}:{{ web_group }} /run/uwsgi; source scl_source rh-python36' +ExecStart=/bin/scl enable rh-python36 "uwsgi --emperor /etc/uwsgi/sites" Restart=always KillSignal=SIGQUIT Type=notify From 1bc45b11064b8773f9b79ad0f90779bbb13b334f Mon Sep 17 00:00:00 2001 From: James Graham Date: Mon, 30 Mar 2020 20:30:37 +0100 Subject: [PATCH 08/18] deploy: Only configure firewall if it's already running --- roles/webserver/tasks/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/webserver/tasks/main.yml b/roles/webserver/tasks/main.yml index cc1504c..aa8c769 100644 --- a/roles/webserver/tasks/main.yml +++ b/roles/webserver/tasks/main.yml @@ -223,6 +223,9 @@ - uwsgi - rh-nginx114-nginx +- name: Populate service facts + service_facts: + - name: Open webserver ports on firewall firewalld: service: '{{ item }}' @@ -232,4 +235,4 @@ loop: - http - https - when: vagrant_dir.stat.exists == False + when: ansible_facts.services['firewalld.service'] is defined and ansible_facts.services['firewalld.service'].state == 'running' From 834fb3c644acd2e19b043bd155973f8c5b32f2fd Mon Sep 17 00:00:00 2001 From: James Graham Date: Mon, 30 Mar 2020 20:52:09 +0100 Subject: [PATCH 09/18] fix: Add login required on all remaining views --- activities/views.py | 9 +++++---- breccia_mapper/views.py | 9 ++++++++- people/views/export.py | 3 ++- people/views/person.py | 5 +++-- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/activities/views.py b/activities/views.py index a2b4a0e..72b2d55 100644 --- a/activities/views.py +++ b/activities/views.py @@ -3,6 +3,7 @@ Views for displaying / manipulating models within the Activities app. """ import json +from django.contrib.auth.mixins import LoginRequiredMixin from django.http import HttpResponse from django.views.generic import DetailView, ListView, View from django.views.generic.detail import SingleObjectMixin @@ -12,7 +13,7 @@ from people import permissions from . import models -class ActivitySeriesListView(ListView): +class ActivitySeriesListView(LoginRequiredMixin, ListView): """ View displaying a list of :class:`ActivitySeries`. """ @@ -21,7 +22,7 @@ class ActivitySeriesListView(ListView): context_object_name = 'activity_series_list' -class ActivitySeriesDetailView(DetailView): +class ActivitySeriesDetailView(LoginRequiredMixin, DetailView): """ View displaying details of a single :class:`ActivitySeries`. """ @@ -30,7 +31,7 @@ class ActivitySeriesDetailView(DetailView): context_object_name = 'activity_series' -class ActivityListView(ListView): +class ActivityListView(LoginRequiredMixin, ListView): """ View displaying a list of :class:`Activity`. """ @@ -38,7 +39,7 @@ class ActivityListView(ListView): template_name = 'activities/activity/list.html' -class ActivityDetailView(DetailView): +class ActivityDetailView(LoginRequiredMixin, DetailView): """ View displaying details of a single :class:`Activity`. """ diff --git a/breccia_mapper/views.py b/breccia_mapper/views.py index 33ed8cd..86480e6 100644 --- a/breccia_mapper/views.py +++ b/breccia_mapper/views.py @@ -1,3 +1,10 @@ +""" +Views belonging to the core of the project. + +These views don't represent any of the models in the apps. +""" + +from django.contrib.auth.mixins import LoginRequiredMixin from django.views.generic import TemplateView @@ -5,5 +12,5 @@ class IndexView(TemplateView): template_name = 'index.html' -class ExportListView(TemplateView): +class ExportListView(LoginRequiredMixin, TemplateView): template_name = 'export.html' diff --git a/people/views/export.py b/people/views/export.py index 343b7f7..6e62b26 100644 --- a/people/views/export.py +++ b/people/views/export.py @@ -1,13 +1,14 @@ import csv import typing +from django.contrib.auth.mixins import LoginRequiredMixin from django.http import HttpResponse from django.views.generic.list import BaseListView from .. import models, serializers -class CsvExportView(BaseListView): +class CsvExportView(LoginRequiredMixin, BaseListView): model = None serializer_class = None diff --git a/people/views/person.py b/people/views/person.py index 1ba28f5..92e7ee9 100644 --- a/people/views/person.py +++ b/people/views/person.py @@ -2,12 +2,13 @@ Views for displaying or manipulating instances of :class:`Person`. """ +from django.contrib.auth.mixins import LoginRequiredMixin from django.views.generic import CreateView, DetailView, ListView, UpdateView from people import forms, models, permissions -class PersonCreateView(CreateView): +class PersonCreateView(LoginRequiredMixin, CreateView): """ View to create a new instance of :class:`Person`. @@ -24,7 +25,7 @@ class PersonCreateView(CreateView): return super().form_valid(form) -class PersonListView(ListView): +class PersonListView(LoginRequiredMixin, ListView): """ View displaying a list of :class:`Person` objects - searchable. """ From 80f7fb08577a255b54f528871f21a1d3b215bf24 Mon Sep 17 00:00:00 2001 From: James Graham Date: Mon, 30 Mar 2020 20:54:15 +0100 Subject: [PATCH 10/18] deploy: Add SSH to firewall allowed services --- roles/webserver/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/webserver/tasks/main.yml b/roles/webserver/tasks/main.yml index aa8c769..ef16b98 100644 --- a/roles/webserver/tasks/main.yml +++ b/roles/webserver/tasks/main.yml @@ -233,6 +233,7 @@ permanent: yes immediate: yes loop: + - ssh - http - https when: ansible_facts.services['firewalld.service'] is defined and ansible_facts.services['firewalld.service'].state == 'running' From 04ae8cb4f655204cfbb09f1c68cc0e3d6c71d108 Mon Sep 17 00:00:00 2001 From: James Graham Date: Tue, 31 Mar 2020 15:43:33 +0100 Subject: [PATCH 11/18] feat: Recursively flatten serializers for CSV export --- people/serializers.py | 3 +++ people/views/export.py | 46 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/people/serializers.py b/people/serializers.py index 85e1c27..bd501d5 100644 --- a/people/serializers.py +++ b/people/serializers.py @@ -31,6 +31,9 @@ class PersonExportSerializer(serializers.ModelSerializer): class RelationshipSerializer(serializers.ModelSerializer): + source = PersonSerializer() + target = PersonSerializer() + class Meta: model = models.Relationship fields = [ diff --git a/people/views/export.py b/people/views/export.py index 6e62b26..8e58ea9 100644 --- a/people/views/export.py +++ b/people/views/export.py @@ -5,6 +5,8 @@ from django.contrib.auth.mixins import LoginRequiredMixin from django.http import HttpResponse from django.views.generic.list import BaseListView +from rest_framework.serializers import BaseSerializer + from .. import models, serializers @@ -12,14 +14,54 @@ class CsvExportView(LoginRequiredMixin, BaseListView): model = None serializer_class = None + @classmethod + def flatten_data(cls, data, + sub_type: typing.Type = dict, + sub_value_accessor: typing.Callable = lambda x: x.items()) -> typing.Dict: + """ + Flatten a dictionary so that subdictionaryies become a series of `key[.subkey[.subsubkey ...]]` entries + in the top level dictionary. + + Works for other data structures (e.g. DRF Serializers) by providing suitable values for the + `sub_type` and `sub_value_accessor` parameters. + + :param data: Dictionary or other data structure to flatten + :param sub_type: Type to recursively flatten + :param sub_value_accessor: Function to access keys and values contained within sub_type. + """ + data_out = {} + + for key, value in sub_value_accessor(data): + if isinstance(value, sub_type): + # Recursively flatten nested structures of type `sub_type` + sub_flattened = cls.flatten_data(value, + sub_type=sub_type, + sub_value_accessor=sub_value_accessor).items() + + # Enter recursively flattened values into result dictionary + for sub_key, sub_value in sub_flattened: + # Keys in result dictionary are of format `key[.subkey[.subsubkey ...]]` + data_out[f'{key}.{sub_key}'] = sub_value + + else: + data_out[key] = value + + return data_out + def render_to_response(self, context: typing.Dict) -> HttpResponse: response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = f'attachment; filename="{self.get_context_object_name(self.object_list)}.csv"' serializer = self.serializer_class(self.get_queryset(), many=True) - writer = csv.DictWriter(response, fieldnames=self.serializer_class.Meta.fields) + columns = self.flatten_data(serializer.child.fields, + sub_type=BaseSerializer, + sub_value_accessor=lambda x: x.fields.items()) + + writer = csv.DictWriter(response, fieldnames=columns) writer.writeheader() - writer.writerows(serializer.data) + + for row in serializer.data: + writer.writerow(self.flatten_data(row)) return response From 76ae447cc60d329102a98581d60cf6a7f85bca23 Mon Sep 17 00:00:00 2001 From: James Graham Date: Wed, 1 Apr 2020 09:57:28 +0100 Subject: [PATCH 12/18] refactor: Move flattening to serializer base class --- people/serializers.py | 72 +++++++++++++++++++++++++++++++++++++++++- people/views/export.py | 47 ++------------------------- 2 files changed, 74 insertions(+), 45 deletions(-) diff --git a/people/serializers.py b/people/serializers.py index bd501d5..3763756 100644 --- a/people/serializers.py +++ b/people/serializers.py @@ -2,11 +2,69 @@ Serialize models to and deserialize from JSON. """ +from collections import OrderedDict +import typing + from rest_framework import serializers from . import models +class FlattenedModelSerializer(serializers.ModelSerializer): + @classmethod + def flatten_data(cls, data, + sub_type: typing.Type = dict, + sub_value_accessor: typing.Callable = lambda x: x.items()) -> typing.OrderedDict: + """ + Flatten a dictionary so that subdictionaries become a series of `key[.subkey[.subsubkey ...]]` entries + in the top level dictionary. + + Works for other data structures (e.g. DRF Serializers) by providing suitable values for the + `sub_type` and `sub_value_accessor` parameters. + + :param data: Dictionary or other data structure to flatten + :param sub_type: Type to recursively flatten + :param sub_value_accessor: Function to access keys and values contained within sub_type. + """ + data_out = OrderedDict() + + for key, value in sub_value_accessor(data): + if isinstance(value, sub_type): + # Recursively flatten nested structures of type `sub_type` + sub_flattened = cls.flatten_data(value, + sub_type=sub_type, + sub_value_accessor=sub_value_accessor).items() + + # Enter recursively flattened values into result dictionary + for sub_key, sub_value in sub_flattened: + # Keys in result dictionary are of format `key[.subkey[.subsubkey ...]]` + data_out[f'{key}.{sub_key}'] = sub_value + + else: + data_out[key] = value + + return data_out + + @property + def column_headers(self) -> typing.Collection: + """ + Get all column headers that will be output by this serializer. + """ + return self.flatten_data(self.fields, + sub_type=serializers.BaseSerializer, + sub_value_accessor=lambda x: x.fields.items()) + + def to_representation(self, instance) -> typing.OrderedDict: + """ + + """ + rep = super().to_representation(instance) + + rep = self.flatten_data(rep) + + return rep + + class PersonSerializer(serializers.ModelSerializer): class Meta: model = models.Person @@ -30,7 +88,7 @@ class PersonExportSerializer(serializers.ModelSerializer): ] -class RelationshipSerializer(serializers.ModelSerializer): +class RelationshipSerializer(FlattenedModelSerializer): source = PersonSerializer() target = PersonSerializer() @@ -41,3 +99,15 @@ class RelationshipSerializer(serializers.ModelSerializer): 'source', 'target', ] + + def to_representation(self, instance): + rep = super().to_representation(instance) + + # try: + # for answer in instance.current_answers.question_answers.all(): + # rep[answer.question.text] = answer.text + # + # except AttributeError: + # pass + + return rep diff --git a/people/views/export.py b/people/views/export.py index 8e58ea9..c4bdc88 100644 --- a/people/views/export.py +++ b/people/views/export.py @@ -5,8 +5,6 @@ from django.contrib.auth.mixins import LoginRequiredMixin from django.http import HttpResponse from django.views.generic.list import BaseListView -from rest_framework.serializers import BaseSerializer - from .. import models, serializers @@ -14,55 +12,16 @@ class CsvExportView(LoginRequiredMixin, BaseListView): model = None serializer_class = None - @classmethod - def flatten_data(cls, data, - sub_type: typing.Type = dict, - sub_value_accessor: typing.Callable = lambda x: x.items()) -> typing.Dict: - """ - Flatten a dictionary so that subdictionaryies become a series of `key[.subkey[.subsubkey ...]]` entries - in the top level dictionary. - - Works for other data structures (e.g. DRF Serializers) by providing suitable values for the - `sub_type` and `sub_value_accessor` parameters. - - :param data: Dictionary or other data structure to flatten - :param sub_type: Type to recursively flatten - :param sub_value_accessor: Function to access keys and values contained within sub_type. - """ - data_out = {} - - for key, value in sub_value_accessor(data): - if isinstance(value, sub_type): - # Recursively flatten nested structures of type `sub_type` - sub_flattened = cls.flatten_data(value, - sub_type=sub_type, - sub_value_accessor=sub_value_accessor).items() - - # Enter recursively flattened values into result dictionary - for sub_key, sub_value in sub_flattened: - # Keys in result dictionary are of format `key[.subkey[.subsubkey ...]]` - data_out[f'{key}.{sub_key}'] = sub_value - - else: - data_out[key] = value - - return data_out - def render_to_response(self, context: typing.Dict) -> HttpResponse: response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = f'attachment; filename="{self.get_context_object_name(self.object_list)}.csv"' serializer = self.serializer_class(self.get_queryset(), many=True) - columns = self.flatten_data(serializer.child.fields, - sub_type=BaseSerializer, - sub_value_accessor=lambda x: x.fields.items()) - - writer = csv.DictWriter(response, fieldnames=columns) + + writer = csv.DictWriter(response, fieldnames=serializer.child.column_headers) writer.writeheader() + writer.writerows(serializer.data) - for row in serializer.data: - writer.writerow(self.flatten_data(row)) - return response From 76270c45722059483385aeb0346fe8660402ab66 Mon Sep 17 00:00:00 2001 From: James Graham Date: Wed, 1 Apr 2020 13:15:48 +0100 Subject: [PATCH 13/18] feat: Export relationships with answers See #27 See #29 Resolves #25 --- people/models/relationship.py | 9 +++++++++ people/serializers.py | 33 +++++++++++++++++++++++---------- people/views/export.py | 5 +++-- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/people/models/relationship.py b/people/models/relationship.py index 9065ac8..c141810 100644 --- a/people/models/relationship.py +++ b/people/models/relationship.py @@ -6,6 +6,7 @@ import typing from django.db import models from django.urls import reverse +from django.utils.text import slugify from .person import Person @@ -47,6 +48,10 @@ class RelationshipQuestion(models.Model): return [ [choice.pk, str(choice)] for choice in self.answers.all() ] + + @property + def slug(self) -> str: + return slugify(self.text) def __str__(self) -> str: return self.text @@ -80,6 +85,10 @@ class RelationshipQuestionChoice(models.Model): order = models.SmallIntegerField(default=0, blank=False, null=False) + @property + def slug(self) -> str: + return slugify(self.text) + def __str__(self) -> str: return self.text diff --git a/people/serializers.py b/people/serializers.py index 3763756..8d820f7 100644 --- a/people/serializers.py +++ b/people/serializers.py @@ -46,13 +46,15 @@ class FlattenedModelSerializer(serializers.ModelSerializer): return data_out @property - def column_headers(self) -> typing.Collection: + def column_headers(self) -> typing.List[str]: """ Get all column headers that will be output by this serializer. """ - return self.flatten_data(self.fields, - sub_type=serializers.BaseSerializer, - sub_value_accessor=lambda x: x.fields.items()) + fields = self.flatten_data(self.fields, + sub_type=serializers.BaseSerializer, + sub_value_accessor=lambda x: x.fields.items()) + + return list(fields) def to_representation(self, instance) -> typing.OrderedDict: """ @@ -99,15 +101,26 @@ class RelationshipSerializer(FlattenedModelSerializer): 'source', 'target', ] + + @property + def column_headers(self) -> typing.List[str]: + headers = super().column_headers + + # Add relationship questions to columns + for question in models.RelationshipQuestion.objects.all(): + headers.append(question.slug) + + return headers def to_representation(self, instance): rep = super().to_representation(instance) - # try: - # for answer in instance.current_answers.question_answers.all(): - # rep[answer.question.text] = answer.text - # - # except AttributeError: - # pass + try: + # Add relationship question answers to data + for answer in instance.current_answers.question_answers.all(): + rep[answer.question.slug] = answer.slug + + except AttributeError: + pass return rep diff --git a/people/views/export.py b/people/views/export.py index c4bdc88..e57ba2b 100644 --- a/people/views/export.py +++ b/people/views/export.py @@ -15,8 +15,9 @@ class CsvExportView(LoginRequiredMixin, BaseListView): def render_to_response(self, context: typing.Dict) -> HttpResponse: response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = f'attachment; filename="{self.get_context_object_name(self.object_list)}.csv"' - - serializer = self.serializer_class(self.get_queryset(), many=True) + + # Force ordering by PK - though this should be default anyway + serializer = self.serializer_class(self.get_queryset().order_by('pk'), many=True) writer = csv.DictWriter(response, fieldnames=serializer.child.column_headers) writer.writeheader() From d02f865952f04ed1308ad23e21cfd7b0ea300e45 Mon Sep 17 00:00:00 2001 From: James Graham Date: Wed, 1 Apr 2020 16:00:22 +0100 Subject: [PATCH 14/18] refactor: Move export into separate app --- breccia_mapper/settings.py | 1 + breccia_mapper/templates/base.html | 2 +- breccia_mapper/urls.py | 5 +- breccia_mapper/views.py | 5 - export/__init__.py | 0 export/apps.py | 5 + export/serializers/__init__.py | 3 + export/serializers/base.py | 65 ++++++++++++ export/serializers/people.py | 66 +++++++++++++ .../templates/export}/export.html | 4 +- export/urls.py | 20 ++++ export/views/__init__.py | 5 + .../views/export.py => export/views/base.py | 13 +-- export/views/people.py | 14 +++ people/serializers.py | 99 +------------------ people/urls.py | 8 -- people/views/__init__.py | 1 - 17 files changed, 188 insertions(+), 128 deletions(-) create mode 100644 export/__init__.py create mode 100644 export/apps.py create mode 100644 export/serializers/__init__.py create mode 100644 export/serializers/base.py create mode 100644 export/serializers/people.py rename {breccia_mapper/templates => export/templates/export}/export.html (84%) create mode 100644 export/urls.py create mode 100644 export/views/__init__.py rename people/views/export.py => export/views/base.py (74%) create mode 100644 export/views/people.py diff --git a/breccia_mapper/settings.py b/breccia_mapper/settings.py index 795ae33..16d6569 100644 --- a/breccia_mapper/settings.py +++ b/breccia_mapper/settings.py @@ -77,6 +77,7 @@ THIRD_PARTY_APPS = [ FIRST_PARTY_APPS = [ 'people', 'activities', + 'export', ] INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + FIRST_PARTY_APPS diff --git a/breccia_mapper/templates/base.html b/breccia_mapper/templates/base.html index ef6c828..3d2ddc0 100644 --- a/breccia_mapper/templates/base.html +++ b/breccia_mapper/templates/base.html @@ -79,7 +79,7 @@ {% if request.user.is_superuser %}
  • - Export + Export
  • diff --git a/breccia_mapper/urls.py b/breccia_mapper/urls.py index b8ed941..2806827 100644 --- a/breccia_mapper/urls.py +++ b/breccia_mapper/urls.py @@ -28,9 +28,8 @@ urlpatterns = [ views.IndexView.as_view(), name='index'), - path('export', - views.ExportListView.as_view(), - name='export'), + path('', + include('export.urls')), path('', include('people.urls')), diff --git a/breccia_mapper/views.py b/breccia_mapper/views.py index 86480e6..272f7c9 100644 --- a/breccia_mapper/views.py +++ b/breccia_mapper/views.py @@ -4,13 +4,8 @@ Views belonging to the core of the project. These views don't represent any of the models in the apps. """ -from django.contrib.auth.mixins import LoginRequiredMixin from django.views.generic import TemplateView class IndexView(TemplateView): template_name = 'index.html' - - -class ExportListView(LoginRequiredMixin, TemplateView): - template_name = 'export.html' diff --git a/export/__init__.py b/export/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/export/apps.py b/export/apps.py new file mode 100644 index 0000000..15c9d19 --- /dev/null +++ b/export/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class ExportConfig(AppConfig): + name = 'export' diff --git a/export/serializers/__init__.py b/export/serializers/__init__.py new file mode 100644 index 0000000..5f4be95 --- /dev/null +++ b/export/serializers/__init__.py @@ -0,0 +1,3 @@ +from . import ( + people +) diff --git a/export/serializers/base.py b/export/serializers/base.py new file mode 100644 index 0000000..0df4a45 --- /dev/null +++ b/export/serializers/base.py @@ -0,0 +1,65 @@ +""" +Serialize models to and deserialize from JSON. +""" + +from collections import OrderedDict +import typing + +from rest_framework import serializers + + +class FlattenedModelSerializer(serializers.ModelSerializer): + @classmethod + def flatten_data(cls, data, + sub_type: typing.Type = dict, + sub_value_accessor: typing.Callable = lambda x: x.items()) -> typing.OrderedDict: + """ + Flatten a dictionary so that subdictionaries become a series of `key[.subkey[.subsubkey ...]]` entries + in the top level dictionary. + + Works for other data structures (e.g. DRF Serializers) by providing suitable values for the + `sub_type` and `sub_value_accessor` parameters. + + :param data: Dictionary or other data structure to flatten + :param sub_type: Type to recursively flatten + :param sub_value_accessor: Function to access keys and values contained within sub_type. + """ + data_out = OrderedDict() + + for key, value in sub_value_accessor(data): + if isinstance(value, sub_type): + # Recursively flatten nested structures of type `sub_type` + sub_flattened = cls.flatten_data(value, + sub_type=sub_type, + sub_value_accessor=sub_value_accessor).items() + + # Enter recursively flattened values into result dictionary + for sub_key, sub_value in sub_flattened: + # Keys in result dictionary are of format `key[.subkey[.subsubkey ...]]` + data_out[f'{key}.{sub_key}'] = sub_value + + else: + data_out[key] = value + + return data_out + + @property + def column_headers(self) -> typing.List[str]: + """ + Get all column headers that will be output by this serializer. + """ + fields = self.flatten_data(self.fields, + sub_type=serializers.BaseSerializer, + sub_value_accessor=lambda x: x.fields.items()) + + return list(fields) + + def to_representation(self, instance) -> typing.OrderedDict: + """ + + """ + rep = super().to_representation(instance) + + rep = self.flatten_data(rep) + + return rep diff --git a/export/serializers/people.py b/export/serializers/people.py new file mode 100644 index 0000000..4ded304 --- /dev/null +++ b/export/serializers/people.py @@ -0,0 +1,66 @@ +import typing + +from rest_framework import serializers + +from people import models + +from . import base + + +class PersonSerializer(serializers.ModelSerializer): + class Meta: + model = models.Person + fields = [ + 'pk', + 'name', + ] + + +class PersonExportSerializer(serializers.ModelSerializer): + class Meta: + model = models.Person + fields = [ + 'pk', + 'name', + 'core_member', + 'gender', + 'age_group', + 'nationality', + 'country_of_residence', + ] + + +class RelationshipSerializer(base.FlattenedModelSerializer): + source = PersonSerializer() + target = PersonSerializer() + + class Meta: + model = models.Relationship + fields = [ + 'pk', + 'source', + 'target', + ] + + @property + def column_headers(self) -> typing.List[str]: + headers = super().column_headers + + # Add relationship questions to columns + for question in models.RelationshipQuestion.objects.all(): + headers.append(question.slug) + + return headers + + def to_representation(self, instance): + rep = super().to_representation(instance) + + try: + # Add relationship question answers to data + for answer in instance.current_answers.question_answers.all(): + rep[answer.question.slug] = answer.slug + + except AttributeError: + pass + + return rep diff --git a/breccia_mapper/templates/export.html b/export/templates/export/export.html similarity index 84% rename from breccia_mapper/templates/export.html rename to export/templates/export/export.html index e0aca40..d8741e9 100644 --- a/breccia_mapper/templates/export.html +++ b/export/templates/export/export.html @@ -24,7 +24,7 @@ Export + href="{% url 'export:person' %}">Export @@ -33,7 +33,7 @@ Export + href="{% url 'export:relationship' %}">Export diff --git a/export/urls.py b/export/urls.py new file mode 100644 index 0000000..c991724 --- /dev/null +++ b/export/urls.py @@ -0,0 +1,20 @@ +from django.urls import path + +from . import views + + +app_name = 'export' + +urlpatterns = [ + path('export', + views.ExportListView.as_view(), + name='index'), + + path('export/people', + views.people.PersonExportView.as_view(), + name='person'), + + path('export/relationships', + views.people.RelationshipExportView.as_view(), + name='relationship'), +] diff --git a/export/views/__init__.py b/export/views/__init__.py new file mode 100644 index 0000000..5efd3f7 --- /dev/null +++ b/export/views/__init__.py @@ -0,0 +1,5 @@ +from .base import ExportListView + +from . import ( + people +) diff --git a/people/views/export.py b/export/views/base.py similarity index 74% rename from people/views/export.py rename to export/views/base.py index e57ba2b..8416e47 100644 --- a/people/views/export.py +++ b/export/views/base.py @@ -3,10 +3,9 @@ import typing from django.contrib.auth.mixins import LoginRequiredMixin from django.http import HttpResponse +from django.views.generic import TemplateView from django.views.generic.list import BaseListView -from .. import models, serializers - class CsvExportView(LoginRequiredMixin, BaseListView): model = None @@ -26,11 +25,5 @@ class CsvExportView(LoginRequiredMixin, BaseListView): return response -class PersonExportView(CsvExportView): - model = models.Person - serializer_class = serializers.PersonExportSerializer - - -class RelationshipExportView(CsvExportView): - model = models.Relationship - serializer_class = serializers.RelationshipSerializer +class ExportListView(LoginRequiredMixin, TemplateView): + template_name = 'export/export.html' diff --git a/export/views/people.py b/export/views/people.py new file mode 100644 index 0000000..77d6999 --- /dev/null +++ b/export/views/people.py @@ -0,0 +1,14 @@ +from . import base +from .. import serializers + +from people import models + + +class PersonExportView(base.CsvExportView): + model = models.person.Person + serializer_class = serializers.people.PersonExportSerializer + + +class RelationshipExportView(base.CsvExportView): + model = models.relationship.Relationship + serializer_class = serializers.people.RelationshipSerializer diff --git a/people/serializers.py b/people/serializers.py index 8d820f7..f11004e 100644 --- a/people/serializers.py +++ b/people/serializers.py @@ -2,71 +2,11 @@ Serialize models to and deserialize from JSON. """ -from collections import OrderedDict -import typing - from rest_framework import serializers from . import models -class FlattenedModelSerializer(serializers.ModelSerializer): - @classmethod - def flatten_data(cls, data, - sub_type: typing.Type = dict, - sub_value_accessor: typing.Callable = lambda x: x.items()) -> typing.OrderedDict: - """ - Flatten a dictionary so that subdictionaries become a series of `key[.subkey[.subsubkey ...]]` entries - in the top level dictionary. - - Works for other data structures (e.g. DRF Serializers) by providing suitable values for the - `sub_type` and `sub_value_accessor` parameters. - - :param data: Dictionary or other data structure to flatten - :param sub_type: Type to recursively flatten - :param sub_value_accessor: Function to access keys and values contained within sub_type. - """ - data_out = OrderedDict() - - for key, value in sub_value_accessor(data): - if isinstance(value, sub_type): - # Recursively flatten nested structures of type `sub_type` - sub_flattened = cls.flatten_data(value, - sub_type=sub_type, - sub_value_accessor=sub_value_accessor).items() - - # Enter recursively flattened values into result dictionary - for sub_key, sub_value in sub_flattened: - # Keys in result dictionary are of format `key[.subkey[.subsubkey ...]]` - data_out[f'{key}.{sub_key}'] = sub_value - - else: - data_out[key] = value - - return data_out - - @property - def column_headers(self) -> typing.List[str]: - """ - Get all column headers that will be output by this serializer. - """ - fields = self.flatten_data(self.fields, - sub_type=serializers.BaseSerializer, - sub_value_accessor=lambda x: x.fields.items()) - - return list(fields) - - def to_representation(self, instance) -> typing.OrderedDict: - """ - - """ - rep = super().to_representation(instance) - - rep = self.flatten_data(rep) - - return rep - - class PersonSerializer(serializers.ModelSerializer): class Meta: model = models.Person @@ -75,22 +15,8 @@ class PersonSerializer(serializers.ModelSerializer): 'name', ] - -class PersonExportSerializer(serializers.ModelSerializer): - class Meta: - model = models.Person - fields = [ - 'pk', - 'name', - 'core_member', - 'gender', - 'age_group', - 'nationality', - 'country_of_residence', - ] - -class RelationshipSerializer(FlattenedModelSerializer): +class RelationshipSerializer(serializers.ModelSerializer): source = PersonSerializer() target = PersonSerializer() @@ -101,26 +27,3 @@ class RelationshipSerializer(FlattenedModelSerializer): 'source', 'target', ] - - @property - def column_headers(self) -> typing.List[str]: - headers = super().column_headers - - # Add relationship questions to columns - for question in models.RelationshipQuestion.objects.all(): - headers.append(question.slug) - - return headers - - def to_representation(self, instance): - rep = super().to_representation(instance) - - try: - # Add relationship question answers to data - for answer in instance.current_answers.question_answers.all(): - rep[answer.question.slug] = answer.slug - - except AttributeError: - pass - - return rep diff --git a/people/urls.py b/people/urls.py index 784522c..eb73fe1 100644 --- a/people/urls.py +++ b/people/urls.py @@ -38,14 +38,6 @@ urlpatterns = [ views.relationship.RelationshipUpdateView.as_view(), name='relationship.update'), - path('people/export', - views.export.PersonExportView.as_view(), - name='person.export'), - - path('relationships/export', - views.export.RelationshipExportView.as_view(), - name='relationship.export'), - path('network', views.network.NetworkView.as_view(), name='network'), diff --git a/people/views/__init__.py b/people/views/__init__.py index 5dd2c46..39dd5c9 100644 --- a/people/views/__init__.py +++ b/people/views/__init__.py @@ -3,7 +3,6 @@ Views for displaying or manipulating models within the `people` app. """ from . import ( - export, network, person, relationship From 7d1aac2021d47513e736c66acd3bfd293d53278d Mon Sep 17 00:00:00 2001 From: James Graham Date: Wed, 1 Apr 2020 16:23:16 +0100 Subject: [PATCH 15/18] refactor: Rename Person serializers --- export/serializers/people.py | 8 ++++---- export/views/people.py | 2 +- people/models/relationship.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/export/serializers/people.py b/export/serializers/people.py index 4ded304..baed3bb 100644 --- a/export/serializers/people.py +++ b/export/serializers/people.py @@ -7,7 +7,7 @@ from people import models from . import base -class PersonSerializer(serializers.ModelSerializer): +class SimplePersonSerializer(serializers.ModelSerializer): class Meta: model = models.Person fields = [ @@ -16,7 +16,7 @@ class PersonSerializer(serializers.ModelSerializer): ] -class PersonExportSerializer(serializers.ModelSerializer): +class PersonSerializer(base.FlattenedModelSerializer): class Meta: model = models.Person fields = [ @@ -31,8 +31,8 @@ class PersonExportSerializer(serializers.ModelSerializer): class RelationshipSerializer(base.FlattenedModelSerializer): - source = PersonSerializer() - target = PersonSerializer() + source = SimplePersonSerializer() + target = SimplePersonSerializer() class Meta: model = models.Relationship diff --git a/export/views/people.py b/export/views/people.py index 77d6999..7f73d5f 100644 --- a/export/views/people.py +++ b/export/views/people.py @@ -6,7 +6,7 @@ from people import models class PersonExportView(base.CsvExportView): model = models.person.Person - serializer_class = serializers.people.PersonExportSerializer + serializer_class = serializers.people.PersonSerializer class RelationshipExportView(base.CsvExportView): diff --git a/people/models/relationship.py b/people/models/relationship.py index c141810..2776f71 100644 --- a/people/models/relationship.py +++ b/people/models/relationship.py @@ -43,7 +43,7 @@ class RelationshipQuestion(models.Model): @property def choices(self) -> typing.List[typing.List[str]]: """ - Convert the :class:`RelationshipQuestionChoices` for this question into Django choices. + Convert the :class:`RelationshipQuestionChoice`s for this question into Django choices. """ return [ [choice.pk, str(choice)] for choice in self.answers.all() From 15aab2eb33ec5f756eb04733cb0211afdfc7cd2c Mon Sep 17 00:00:00 2001 From: James Graham Date: Wed, 1 Apr 2020 16:48:23 +0100 Subject: [PATCH 16/18] feat: Add Activity export See #27 See #29 Resolves #24 --- export/serializers/__init__.py | 1 + export/serializers/activities.py | 52 +++++++++++++++++++++++++++++ export/templates/export/export.html | 9 +++++ export/urls.py | 4 +++ export/views/__init__.py | 1 + export/views/activities.py | 9 +++++ 6 files changed, 76 insertions(+) create mode 100644 export/serializers/activities.py create mode 100644 export/views/activities.py diff --git a/export/serializers/__init__.py b/export/serializers/__init__.py index 5f4be95..104caf6 100644 --- a/export/serializers/__init__.py +++ b/export/serializers/__init__.py @@ -1,3 +1,4 @@ from . import ( + activities, people ) diff --git a/export/serializers/activities.py b/export/serializers/activities.py new file mode 100644 index 0000000..a2069cd --- /dev/null +++ b/export/serializers/activities.py @@ -0,0 +1,52 @@ +from rest_framework import serializers + +from activities import models + +from . import base + + +class ActivityTypeSerializer(serializers.ModelSerializer): + class Meta: + model = models.ActivityType + fields = [ + 'pk', + 'name', + ] + + +class ActivityMediumSerializer(serializers.ModelSerializer): + class Meta: + model = models.ActivityMedium + fields = [ + 'pk', + 'name', + ] + + +class ActivitySeriesSerializer(serializers.ModelSerializer): + type = ActivityTypeSerializer() + medium = ActivityMediumSerializer() + + class Meta: + model = models.ActivitySeries + fields = [ + 'pk', + 'name', + 'type', + 'medium', + ] + + +class ActivitySerializer(base.FlattenedModelSerializer): + series = ActivitySeriesSerializer() + type = ActivityTypeSerializer() + medium = ActivityMediumSerializer() + + class Meta: + model = models.Activity + fields = [ + 'name', + 'series', + 'type', + 'medium', + ] diff --git a/export/templates/export/export.html b/export/templates/export/export.html index d8741e9..e4aa7f0 100644 --- a/export/templates/export/export.html +++ b/export/templates/export/export.html @@ -36,6 +36,15 @@ href="{% url 'export:relationship' %}">Export + + + Activities + + + Export + + diff --git a/export/urls.py b/export/urls.py index c991724..7f99467 100644 --- a/export/urls.py +++ b/export/urls.py @@ -17,4 +17,8 @@ urlpatterns = [ path('export/relationships', views.people.RelationshipExportView.as_view(), name='relationship'), + + path('export/activities', + views.activities.ActivityExportView.as_view(), + name='activity'), ] diff --git a/export/views/__init__.py b/export/views/__init__.py index 5efd3f7..da2cdd2 100644 --- a/export/views/__init__.py +++ b/export/views/__init__.py @@ -1,5 +1,6 @@ from .base import ExportListView from . import ( + activities, people ) diff --git a/export/views/activities.py b/export/views/activities.py new file mode 100644 index 0000000..8a74f78 --- /dev/null +++ b/export/views/activities.py @@ -0,0 +1,9 @@ +from . import base +from .. import serializers + +from activities import models + + +class ActivityExportView(base.CsvExportView): + model = models.Activity + serializer_class = serializers.activities.ActivitySerializer From db96e654850cbed0299d7a376e44cda2c3e7f2b1 Mon Sep 17 00:00:00 2001 From: James Graham Date: Wed, 1 Apr 2020 17:05:04 +0100 Subject: [PATCH 17/18] fix: Type annotation not available in py36 --- export/serializers/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/export/serializers/base.py b/export/serializers/base.py index 0df4a45..b7be1c6 100644 --- a/export/serializers/base.py +++ b/export/serializers/base.py @@ -12,7 +12,7 @@ class FlattenedModelSerializer(serializers.ModelSerializer): @classmethod def flatten_data(cls, data, sub_type: typing.Type = dict, - sub_value_accessor: typing.Callable = lambda x: x.items()) -> typing.OrderedDict: + sub_value_accessor: typing.Callable = lambda x: x.items()) -> OrderedDict: """ Flatten a dictionary so that subdictionaries become a series of `key[.subkey[.subsubkey ...]]` entries in the top level dictionary. @@ -54,7 +54,7 @@ class FlattenedModelSerializer(serializers.ModelSerializer): return list(fields) - def to_representation(self, instance) -> typing.OrderedDict: + def to_representation(self, instance) -> OrderedDict: """ """ From 6418c8c96a91efd320da260e2ad4904e0bdfea41 Mon Sep 17 00:00:00 2001 From: James Graham Date: Thu, 2 Apr 2020 10:56:29 +0100 Subject: [PATCH 18/18] fix: Fix broken Activity export Unexpected Series field no longer added when Activity does not belong to a series --- export/serializers/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export/serializers/base.py b/export/serializers/base.py index b7be1c6..f8e359c 100644 --- a/export/serializers/base.py +++ b/export/serializers/base.py @@ -38,7 +38,7 @@ class FlattenedModelSerializer(serializers.ModelSerializer): # Keys in result dictionary are of format `key[.subkey[.subsubkey ...]]` data_out[f'{key}.{sub_key}'] = sub_value - else: + elif value is not None: data_out[key] = value return data_out