Merge pull request #118 from Southampton-RSG/docker

Switch to Docker for deployment

.dockerignore

@@ -0,0 +1,17 @@
+.dbbackup/
+.idea/
+.mypy_cache/
+.vagrant/
+.venv/
+venv/
+.vscode/
+
+Caddyfile
+docker-compose.yml
+settings.ini
+mail.log/
+/roles/
+/static/
+*.sqlite3*
+*.log*
+deployment*

.gitignore

@@ -21,5 +21,6 @@ deployment-key*
 # Deployment
 /.dbbackup/
 .vagrant/
+/custom
 staging.yml
 production.yml

Caddyfile

@@ -0,0 +1,15 @@
+:80 :443 {
+    root * /srv
+    file_server
+
+    @proxy_paths {
+        not path /static/*
+    }
+
+    reverse_proxy @proxy_paths http://web:8000
+
+    log {
+        output stderr
+        format single_field common_log
+    }
+}

Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.8-slim
+
+RUN groupadd -r mapper && useradd --no-log-init -r -g mapper mapper
+
+WORKDIR /app
+
+COPY requirements.txt ./
+
+RUN pip install --no-cache-dir --upgrade pip \
+    && pip install --no-cache-dir -r /app/requirements.txt gunicorn
+
+COPY . ./
+
+# USER mapper
+
+ENTRYPOINT [ "/app/entrypoint.sh" ]
+CMD [ "gunicorn", "-w", "2", "-b", "0.0.0.0:8000", "breccia_mapper.wsgi" ]

deploy/README.md

@@ -0,0 +1,54 @@
+# BRECcIA Mapper Deployment
+
+BRECcIA Mapper is intended to be deployed using Ansible and Docker.
+It has been tested on RHEL7 and RHEL8, though with minor modification to the Ansible playbook it is expected to deploy correctly on other Linux variants (e.g. Ubuntu).
+
+## Development Deployment
+
+Prerequisites:
+
+- [Vagrant](https://www.vagrantup.com/)
+- [Ansible](https://www.ansible.com/)
+
+Using Vagrant, we can create a virtual machine and deploy BRECcIA Mapper using the same provisioning scripts as a production deployment.
+To deploy a local development version of BRECcIA Mapper inside a virtual machine, use:
+
+```
+vagrant up
+```
+
+Once this virtual machine has been created, to redeploy use:
+
+```
+vagrant provision
+```
+
+And to stop the virtual machine use:
+
+```
+vagrant halt
+```
+
+For further commands see the [Vagrant documentation](https://www.vagrantup.com/docs/cli).
+
+## Production Deployment
+
+Prerequisites:
+
+- [Ansible](https://www.ansible.com/)
+
+To perform a production deployment of BRECcIA Mapper:
+
+1. Copy the `inventory.example.yml` to `inventory.yml`
+2. Edit this file:
+  - Use your server's hostname instead of `example.com`
+  - Disable debugging
+  - Replace the secret key with some text known only to you
+3. Run the Ansible playbook with this inventory file using:
+
+```
+ansible-playbook playbook.yml -i inventory.yml -K -k -u <SSH username>
+```
+
+This will ask for your SSH and sudo passwords for the server, before deploying.
+To redeploy updates, the same command can be run again - it's safe to redeploy on top of an existing deployment.

deploy/Vagrantfile

@@ -8,17 +8,23 @@
 Vagrant.configure("2") do |config|
   # Every Vagrant development environment requires a box. You can search for
   # boxes at https://vagrantcloud.com/search.
-  config.vm.box = "centos/7"
+  config.vm.box = "generic/rocky8"
 
   # Create a forwarded port mapping which allows access to a specific port
   # within the machine from a port on the host machine and only allow access
   # via 127.0.0.1 to disable public access
-  config.vm.network "forwarded_port", guest: 80, host: 8888, host_ip: "127.0.0.1"
+  config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1"
-  config.vm.network "forwarded_port", guest: 443, host: 8889, host_ip: "127.0.0.1"
 
   # Provision VM using Ansible playbook
   config.vm.provision "ansible" do |ansible|
     ansible.verbose = "v"
     ansible.playbook = "playbook.yml"
+    ansible.host_vars = {
+      "default" => {
+        "deploy_environment" => "vagrant",
+        "django_debug" => 1,
+        "django_secret_key" => "debug_only_g62WlORMbo8iAcV7vKCKBQ=="
+      }
+    }
   end
 end

deploy/inventory.example.yml

@@ -0,0 +1,5 @@
+all:
+  hosts:
+    example.com:
+      django_debug: 1
+      django_secret_key: debug_only_g62WlORMbo8iAcV7vKCKBQ==

deploy/playbook.yml

@@ -0,0 +1,86 @@
+---
+- hosts: all
+  become_user: root
+  become_method: sudo
+  become: yes
+
+  pre_tasks:
+    - name: Check if running under Vagrant
+      stat:
+        path: /vagrant
+      register: vagrant_dir
+
+  vars:
+    project_name: mapper
+    project_dir: /srv/{{ project_name }}
+    project_src_dir: "{{ project_dir }}/src"
+
+  tasks:
+    - name: Vagrant specific tasks
+      block:
+      - name: Add Docker repository
+        get_url:
+          url: https://download.docker.com/linux/centos/docker-ce.repo
+          dest: '/etc/yum.repos.d/docker-ce.repo'
+      when: deploy_environment is defined and deploy_environment == "vagrant"
+
+    - name: Install system dependencies
+      ansible.builtin.yum:
+        name:
+          - git
+          - docker-ce
+          - docker-ce-cli
+          - containerd.io
+          - docker-compose-plugin
+        state: present
+
+    # - name: Update system packages
+    #   ansible.builtin.yum:
+    #     name: '*'
+    #     state: latest
+
+    - name: Clone / update from source repos
+      ansible.builtin.git:
+        repo: 'https://github.com/Southampton-RSG/breccia-mapper.git'
+        dest: '{{ project_src_dir }}'
+        version: docker
+        accept_hostkey: yes
+
+    - name: Copy template files
+      ansible.builtin.template:
+        src: '{{ item }}.j2'
+        dest: '{{ project_dir }}/{{ item }}'
+        mode: 0600
+      loop:
+        - Caddyfile
+        - docker-compose.yml
+
+    - name: Create database file
+      ansible.builtin.file:
+        path: "{{ project_dir }}/db.sqlite3"
+        state: touch
+
+    - name: Start Docker
+      ansible.builtin.systemd:
+        name: docker
+        state: started
+        enabled: yes
+
+    - name: Pull latest docker images
+      ansible.builtin.command:
+        chdir: "{{ project_dir }}"
+        cmd: docker compose pull {{ item }}
+      loop:
+        - caddy
+
+    - name: Build custom images
+      ansible.builtin.command:
+        chdir: "{{ project_dir }}"
+        cmd: docker compose build {{ item }}
+      loop:
+        - web
+
+    - name: Start containers
+      ansible.builtin.command:
+        chdir: "{{ project_dir }}"
+        cmd: docker compose up -d

deploy/templates/Caddyfile.j2

@@ -0,0 +1,15 @@
+http://* {
+    root * /srv
+    file_server
+
+    @proxy_paths {
+        not path /static/*
+    }
+
+    reverse_proxy @proxy_paths http://web:8000
+
+    log {
+        output stderr
+        format single_field common_log
+    }
+}

deploy/templates/docker-compose.yml.j2

@@ -0,0 +1,35 @@
+version: '3.1'
+
+services:
+  web:
+    image: breccia-mapper
+    build: {{ project_src_dir }}
+    ports:
+      - 8000:8000
+    environment:
+      DEBUG: {{ django_debug }}
+      DATABASE_URL: sqlite:////app/db.sqlite3
+      SECRET_KEY: {{ django_secret_key }}
+    volumes:
+      - {{ project_dir }}/db.sqlite3:/app/db.sqlite3:z
+      - static_files:/app/static
+
+  caddy:
+    image: caddy:2
+    restart: unless-stopped
+    ports:
+      - 80:80
+      - 443:443
+    volumes:
+      - ./Caddyfile:/etc/caddy/Caddyfile:z
+      # Caddy serves static files collected by Django
+      - static_files:/srv/static:ro
+      - caddy_data:/data
+      - caddy_config:/config
+    depends_on:
+      - web
+
+volumes:
+  caddy_data:
+  caddy_config:
+  static_files:

docker-compose.yml

@@ -0,0 +1,35 @@
+version: '3.1'
+
+services:
+  web:
+    image: breccia-mapper
+    build: .
+    ports:
+      - 8000:8000
+    environment:
+      DEBUG: ${DJANGO_DEBUG}
+      DATABASE_URL: sqlite:////app/db.sqlite3
+      SECRET_KEY: ${DJANGO_SECRET_KEY}
+    volumes:
+      - ./db.sqlite3:/app/db.sqlite3:z
+      - static_files:/app/static
+
+  caddy:
+    image: caddy:2
+    restart: unless-stopped
+    ports:
+      - 80:80
+      - 443:443
+    volumes:
+      - ./Caddyfile:/etc/caddy/Caddyfile:z
+      # Caddy serves static files collected by Django
+      - static_files:/srv/static:ro
+      - caddy_data:/data
+      - caddy_config:/config
+    depends_on:
+      - web
+
+volumes:
+  caddy_data:
+  caddy_config:
+  static_files:

entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eo pipefail
+
+python manage.py migrate
+python manage.py collectstatic --no-input
+
+exec "$@"

playbook.yml

@@ -1,18 +0,0 @@
----
-- hosts: all
-  become_user: root
-  become_method: sudo
-  become: yes
-
-  pre_tasks:
-    - name: Check if running under Vagrant
-      stat:
-        path: /vagrant
-      register: vagrant_dir
-
-  roles:
-    - database
-    - webserver
-
-  vars:
-    ansible_python_interpreter: python2

requirements.txt

@@ -21,7 +21,7 @@ isort==4.3.21
 jsonfield==3.1.0
 lazy-object-proxy==1.4.3
 mccabe==0.6.1
-mysqlclient==1.4.6
+# mysqlclient==1.4.6
 pep8-naming==0.4.1
 prospector==1.2.0
 pycodestyle==2.4.0

roles/database/tasks/main.yml

@@ -1,37 +0,0 @@
----
-- name: Update system packages
-  yum:
-    name: '*'
-    state: latest
-
-- name: Install system prerequisites
-  yum:
-    name: '{{ packages }}'
-    state: latest
-  vars:
-    packages:
-      - mariadb
-      - mariadb-devel
-      - mariadb-server
-      - python
-      # For Ansible - not used at runtime
-      - MySQL-python
-
-- name: Restart database server
-  service:
-    name: mariadb
-    state: restarted
-    enabled: yes
-    daemon_reload: yes
-
-- name: Create database
-  mysql_db:
-    name: '{{ db_name }}'
-    state: present
-
-- name: Create database user
-  mysql_user:
-    name: '{{ db_user }}'
-    password: '{{ db_pass }}'
-    state: present
-    priv: '{{ db_name }}.*:ALL'

roles/defaults/main.yml

@@ -1,4 +0,0 @@
----
-db_name: 'breccia'
-db_user: 'breccia'
-db_pass: 'breccia'

roles/webserver/defaults/main.yml

@@ -1,22 +0,0 @@
----
-deploy_mode_dict:
-  1: Production
-  2: Staging
-  3: Development
-deploy_mode: 3
-
-secret_key: '{{ lookup("password", "/dev/null") }}'
-
-parent_project_name: 'BRECcIA'
-project_name: 'breccia-mapper'
-project_full_name: 'breccia_mapper'
-project_dir: '/var/www/{{ project_name }}'
-venv_dir: '{{ project_dir }}/venv'
-web_user: nginx
-web_group: nginx
-db_name: '{{ project_name }}'
-db_user: 'breccia'
-db_pass: 'breccia'
-
-display_short_name: 'BRECcIA'
-display_long_name: 'BRECcIA Mapper'

roles/webserver/tasks/main.yml

@@ -1,255 +0,0 @@
----
-- name: Test connection
-  ping:
-
-- name: Enable EPEL
-  yum:
-    name: epel-release
-    state: latest
-
-- name: Update system packages
-  yum:
-    name: '*'
-    state: latest
-
-- name: Enable RedHat Software Collections - RHEL
-  rhsm_repository:
-    name: rhel-server-rhscl-7-rpms
-  when: ansible_distribution == "RedHat"
-
-- name: Enable RedHat Software Collections - CentOS
-  yum:
-    name: centos-release-scl
-    state: latest
-  when: ansible_distribution == "CentOS"
-
-- name: Install system prerequisites
-  yum:
-    name: '{{ packages }}'
-    state: latest
-  vars:
-    packages:
-      - gcc
-      - git
-      - rh-nginx114
-      - rh-python36
-      - policycoreutils-python
-      - python
-      - python-setuptools
-      - python2-cryptography
-
-- name: (Vagrant only) Clone / update from local repo
-  git:
-    repo: '/vagrant'
-    dest: '{{ project_dir }}'
-  when: vagrant_dir.stat.exists == True
-
-- name: (Vagrant only) Copy local settings file
-  copy:
-    src: '{{ settings_file | default("settings.ini") }}'
-    dest: '{{ project_dir }}/settings.ini'
-    owner: '{{ web_user }}'
-    group: '{{ web_group }}'
-    mode: 0600
-  when: vagrant_dir.stat.exists == True
-
-- name: (Vagrant only) Add DB to settings file
-  ini_file:
-    path: '{{ project_dir }}/settings.ini'
-    section: settings
-    option: DATABASE_URL
-    value: 'mysql://{{ db_user }}:{{ db_pass }}@localhost:3306/{{ db_name }}'
-  when: vagrant_dir.stat.exists == True
-
-- name: Copy deploy key
-  copy:
-    src: '{{ deployment_keyfile }}'
-    dest: '/tmp/deployment-key'
-    mode: 0600
-  when: vagrant_dir.stat.exists == False and deployment_keyfile is defined
-
-- name: Clone / update from source repo
-  git:
-    repo: 'git@github.com:Southampton-RSG/breccia-mapper.git'
-    dest: '{{ project_dir }}'
-    key_file: '{{ "/tmp/deployment-key" if deployment_keyfile is defined else None }}'
-    version: '{{ branch | default ("master") }}'
-    accept_hostkey: yes
-  when: vagrant_dir.stat.exists == False
-
-- name: Copy customisation deploy key
-  copy:
-    src: '{{ customisation_repo_keyfile }}'
-    dest: '/tmp/deployment-key-customisation'
-    mode: 0600
-  when: customisation_repo_keyfile is defined
-
-- name: Clone / update from customisation repo
-  git:
-    repo: '{{ customisation_repo }}'
-    dest: '{{ project_dir }}/custom'
-    key_file: '{{ "/tmp/deployment-key-customisation" if customisation_repo_keyfile is defined else None }}'
-    version: '{{ branch | default ("master") }}'
-    accept_hostkey: yes
-  when: customisation_repo is defined
-
-- name: Copy and populate settings template
-  template:
-    src: 'settings.j2'
-    dest: '{{ project_dir }}/settings.ini'
-    owner: '{{ web_user }}'
-    group: '{{ web_group }}'
-    mode: 0600
-  when: vagrant_dir.stat.exists == False
-
-- name: Set ownership of source directory
-  file:
-    path: '{{ project_dir }}'
-    owner: '{{ web_user }}'
-    group: '{{ web_group }}'
-    recurse: yes
-
-- name: Create venv
-  shell: |
-    source scl_source enable rh-python36
-    python3 -m venv {{ venv_dir }}
-
-- name: Install pip requirements
-  pip:
-    requirements: '{{ project_dir }}/requirements.txt'
-    virtualenv: '{{ venv_dir }}'
-
-- name: Create static directory
-  file:
-    path: '{{ project_dir }}/static'
-    state: directory
-    owner: '{{ web_user }}'
-    group: '{{ web_group }}'
-    mode: 0755
-
-- name: Run Django setup stages
-  django_manage:
-    command: '{{ item }}'
-    app_path: '{{ project_dir }}'
-    virtualenv: '{{ venv_dir }}'
-  become_user: '{{ web_user }}'
-  with_items:
-    - dbbackup
-    - migrate
-    - collectstatic
-
-- name: Apply SELinux type
-  file:
-    path: '{{ project_dir }}/static'
-    state: directory
-    setype: httpd_sys_content_t
-
-- name: (Not production) Set SELinux permissive mode
-  selinux_permissive:
-    name: httpd_t
-    permissive: yes
-  when: deploy_mode > 1
-
-- name: Install uWSGI
-  shell: |
-    source scl_source enable rh-python36
-    pip3 install uwsgi
-
-- name: Setup uWSGI config
-  file:
-    path: /etc/uwsgi/sites
-    state: directory
-    mode: 0755
-
-- name: Setup uWSGI service
-  template:
-    src: uwsgi-service.j2
-    dest: /etc/systemd/system/uwsgi.service
-
-- name: Ensure uWSGI running
-  service:
-    name: uwsgi
-    state: started
-    enabled: yes
-    daemon_reload: yes
-
-- name: Copy web config files
-  template:
-    src: uwsgi-site.j2
-    dest: '/etc/uwsgi/sites/{{ project_name }}.ini'
-
-- name: Generate self-signed SSL certificate
-  block:
-    - name: Create directories
-      file:
-        path: "{{ item }}"
-        state: directory
-      with_items:
-        - /etc/ssl
-        - /etc/ssl/crt
-        - /etc/ssl/private
-        - /etc/ssl/csr
-
-    - name: Create keys
-      openssl_privatekey:
-        path: /etc/ssl/private/{{ inventory_hostname }}.pem
-        owner: '{{ web_user }}'
-        group: '{{ web_user }}'
-
-    - name: Create Certificate Signing Request (CSR)
-      openssl_csr:
-        path: /etc/ssl/csr/{{ inventory_hostname }}.csr
-        privatekey_path: /etc/ssl/private/{{ inventory_hostname }}.pem
-        common_name: "{{ inventory_hostname }}"
-        owner: '{{ web_user }}'
-        group: '{{ web_user }}'
-
-    - name: Generate certificate
-      openssl_certificate:
-        path: /etc/ssl/crt/{{ inventory_hostname }}.crt
-        privatekey_path: /etc/ssl/private/{{ inventory_hostname }}.pem
-        csr_path: /etc/ssl/csr/{{ inventory_hostname }}.csr
-        provider: selfsigned
-        owner: '{{ web_user }}'
-        group: '{{ web_user }}'
-
-    - name: Copy Nginx site
-      template:
-        src: nginx-site-ssl.j2
-        dest: '/etc/opt/rh/rh-nginx114/nginx/conf.d/{{ project_name }}-ssl.conf'
-        owner: '{{ web_user }}'
-        group: '{{ web_group }}'
-
-  when: deploy_mode > 1
-
-- name: Copy Nginx site
-  template:
-    src: nginx-site.j2
-    dest: '/etc/opt/rh/rh-nginx114/nginx/conf.d/{{ project_name }}.conf'
-    owner: '{{ web_user }}'
-    group: '{{ web_group }}'
-
-- name: Restart uWSGI and Nginx
-  service:
-    name: "{{ item }}"
-    state: restarted
-    enabled: yes
-    daemon_reload: yes
-  with_items:
-    - uwsgi
-    - rh-nginx114-nginx
-
-- name: Populate service facts
-  service_facts:
-
-- name: Open webserver ports on firewall
-  firewalld:
-    service: '{{ item }}'
-    state: enabled
-    permanent: yes
-    immediate: yes
-  loop:
-    - ssh
-    - http
-    - https
-  when: ansible_facts.services['firewalld.service'] is defined and ansible_facts.services['firewalld.service'].state == 'running'

roles/webserver/templates/nginx-site-ssl.j2

@@ -1,28 +0,0 @@
-server {
-    # HTTP/2 allows requests to be pipelined within a single connection
-    listen 443 ssl http2;
-    server_name {{ inventory_hostname }} localhost 127.0.0.1;
-
-    ssl_certificate /etc/ssl/crt/{{ inventory_hostname }}.crt;
-    ssl_certificate_key /etc/ssl/private/{{ inventory_hostname }}.pem;
-    ssl_protocols TLSv1.2;
-
-    # Cache and tickets improve performance by ~10% on small requests
-    ssl_session_cache shared:SSL:1m;
-    ssl_session_timeout 4h;
-    ssl_session_tickets on;
-
-    location /favicon.ico {
-        alias {{ project_dir }}/static/img/favicon.ico;
-    }
-
-    location /static/ {
-        alias {{ project_dir }}/static/;
-    }
-
-    location / {
-        include     uwsgi_params;
-        uwsgi_pass  unix:/run/uwsgi/{{ project_name }}.sock;
-        uwsgi_buffers 256 16k;
-    }
-}

roles/webserver/templates/nginx-site.j2

@@ -1,17 +0,0 @@
-server {
-    listen 80;
-    server_name {{ inventory_hostname }} localhost 127.0.0.1;
-
-    location /favicon.ico {
-        alias {{ project_dir }}/static/img/favicon.ico;
-    }
-
-    location /static/ {
-        alias {{ project_dir }}/static/;
-    }
-
-    location / {
-        include     uwsgi_params;
-        uwsgi_pass  unix:/run/uwsgi/{{ project_name }}.sock;
-    }
-}

roles/webserver/templates/settings.j2

@@ -1,30 +0,0 @@
-# Template populated on {{ template_run_date }}
-[settings]
-
-SECRET_KEY={{ secret_key }}
-DEBUG={{ "True" if deploy_mode > 1 else "False" }}
-DATABASE_URL=mysql://{{ db_user }}:{{ db_pass }}@localhost:3306/{{ db_name }}
-
-{% if allowed_hosts is defined %}
-ALLOWED_HOSTS={% for h in allowed_hosts %}{{ h }},{% endfor %}
-{% else %}
-ALLOWED_HOSTS={{ inventory_hostname }},localhost,127.0.0.1
-{% endif %}
-
-PARENT_PROJECT_NAME={{ parent_project_name }}
-PROJECT_SHORT_NAME={{ display_short_name }}
-PROJECT_LONG_NAME={{ display_long_name }}
-
-{% if email_host is defined %}
-EMAIL_HOST={{ email_host }}
-{% endif %}
-{% if default_from_email is defined %}
-DEFAULT_FROM_EMAIL={{ default_from_email }}
-{% endif %}
-{% if email_port is defined %}
-EMAIL_PORT={{ email_port }}
-{% endif %}
-
-{% if google_maps_api_key is defined %}
-GOOGLE_MAPS_API_KEY={{ google_maps_api_key }}
-{% endif %}

roles/webserver/templates/uwsgi-service.j2

@@ -1,13 +0,0 @@
-[Unit]
-Description=uWSGI Emperor Service
-
-[Service]
-ExecStartPre=/bin/bash -c 'mkdir -p /run/uwsgi; chown {{ web_user }}:{{ web_group }} /run/uwsgi; source scl_source rh-python36'
-ExecStart=/bin/scl enable rh-python36 "uwsgi --emperor /etc/uwsgi/sites"
-Restart=always
-KillSignal=SIGQUIT
-Type=notify
-NotifyAccess=all
-
-[Install]
-WantedBy=multi-user.target

roles/webserver/templates/uwsgi-site.j2

@@ -1,19 +0,0 @@
-[uwsgi]
-project = {{ project_name }}
-uid = {{ web_user }}
-gid = {{ web_group }}
-base = /var/www
-
-chdir = %(base)/%(project)
-home = {{ venv_dir }}
-module = {{ project_full_name }}.wsgi:application
-logto = %(chdir)/%(project).log
-
-master = true
-processes = 2
-listen = 128
-
-socket = /run/uwsgi/%(project).sock
-chown-socket = %(uid):{{ web_group }}
-chmod-socket = 660
-vacuum = true