Import opwen-webapp code

Merge pull request #343 from ascoderu/monorepo

.dockerignore

@@ -4,10 +4,20 @@
 !*.yml
 !docker/
 !helm/
+!opwen_email_client/
 !opwen_email_server/
 !opwen_statuspage/
 !tests/
 !makefile
 !requirements*.txt
 !setup.cfg
 !.travis.sh
+!install.py
+!manage.py
+!MANIFEST.in
+!setup.py
+!README.rst
+!package.json
+!yarn.lock
+!Gruntfile.js
+!babel.cfg

.env

@@ -3,6 +3,7 @@ APP_PORT=8080
 BUILD_TAG=development
 BUILD_TARGET=builder
 DOCKER_REPO=ascoderu
+WEBAPP_WORKERS=1
 SERVER_WORKERS=1
 QUEUE_WORKERS=1
 NGINX_WORKERS=1
@@ -14,7 +15,6 @@ LOKOLE_EMAIL_SERVER_QUEUES_NAMESPACE=
 LOKOLE_SENDGRID_KEY=
 LOKOLE_RESOURCE_SUFFIX=
 REGISTRATION_CREDENTIALS=admin:password
-WEBAPP_VERSION=0.5.12
 
 CLOUDBROWSER_PORT=10001
 AZURITE_PORT=10000

.gitignore

@@ -15,3 +15,15 @@ htmlcov/
 dive.log
 
 serviceprincipal.json
+
+node_modules/
+opwen_email_client/webapp/static/css/*.min.css
+opwen_email_client/webapp/static/js/*.min.js
+opwen_email_client/webapp/static/fonts
+opwen_email_client/webapp/static/flags
+
+venv/
+requirements.txt.out
+*.mo
+*.pot
+/state/

.prettierignore

@@ -0,0 +1,2 @@
+*.min.js
+

.travis.yml

@@ -39,6 +39,7 @@ after_script: ./.travis.sh after_script
 before_deploy: ./.travis.sh before_deploy
 deploy:
   - provider: script
+    skip-cleanup: true
     script: ./.travis.sh deploy
     on:  # yamllint disable rule:truthy
       repo: ascoderu/opwen-cloudserver

Gruntfile.js

@@ -0,0 +1,133 @@
+module.exports = function(grunt) {
+
+  grunt.loadNpmTasks('grunt-contrib-copy');
+  grunt.loadNpmTasks('grunt-contrib-cssmin');
+  grunt.loadNpmTasks('grunt-contrib-uglify');
+
+  grunt.initConfig({
+    pkg: grunt.file.readJSON('package.json'),
+
+    copy: {
+      bower: {
+        files: [
+          {
+            expand: true,
+            flatten: true,
+            src: ['node_modules/@bower_components/components-font-awesome/fonts/*'],
+            dest: 'opwen_email_client/webapp/static/fonts/',
+            filter: 'isFile'
+          },
+          {
+            expand: true,
+            flatten: true,
+            src: [
+              'node_modules/@bower_components/flag-icon-css/flags/4x3/ca.svg',
+              'node_modules/@bower_components/flag-icon-css/flags/4x3/cd.svg',
+              'node_modules/@bower_components/flag-icon-css/flags/4x3/pt.svg',
+              'node_modules/@bower_components/flag-icon-css/flags/4x3/tz.svg',
+              'node_modules/@bower_components/flag-icon-css/flags/4x3/fr.svg',
+            ],
+            dest: 'opwen_email_client/webapp/static/flags/4x3/'
+          },
+          {
+            expand: true,
+            flatten: true,
+            src: [
+              'node_modules/@bower_components/jquery/dist/jquery.min.js',
+              'node_modules/@bower_components/bootstrap/dist/js/bootstrap.min.js',
+              'node_modules/@bower_components/bootstrap-fileinput/js/fileinput.min.js',
+              'node_modules/@bower_components/bootstrap3-wysihtml5-bower/dist/bootstrap3-wysihtml5.all.min.js',
+            ],
+            dest: 'opwen_email_client/webapp/static/js/'
+          },
+          {
+            expand: true,
+            flatten: true,
+            src: [
+              'node_modules/@bower_components/components-font-awesome/css/font-awesome.min.css',
+              'node_modules/@bower_components/flag-icon-css/css/flag-icon.min.css',
+              'node_modules/@bower_components/bootstrap/dist/css/bootstrap.min.css',
+              'node_modules/@bower_components/bootstrap-fileinput/css/fileinput.min.css',
+              'node_modules/@bower_components/bootstrap3-wysihtml5-bower/dist/bootstrap3-wysihtml5.min.css',
+            ],
+            dest: 'opwen_email_client/webapp/static/css/'
+          },
+        ]
+      }
+    },
+
+    cssmin: {
+      options: {
+        shorthandCompacting: false,
+        roundingPrecision: -1
+      },
+      target: {
+        files: {
+          'opwen_email_client/webapp/static/css/_base_email.min.css': [
+            'opwen_email_client/webapp/static/css/_base_email.css',
+          ],
+          'opwen_email_client/webapp/static/css/about.min.css': [
+            'opwen_email_client/webapp/static/css/about.css',
+          ],
+          'opwen_email_client/webapp/static/css/home.min.css': [
+            'opwen_email_client/webapp/static/css/home.css',
+          ],
+          'opwen_email_client/webapp/static/css/settings.min.css': [
+            'opwen_email_client/webapp/static/css/settings.css',
+          ],
+          'opwen_email_client/webapp/static/css/email_new.min.css': [
+            'opwen_email_client/webapp/static/css/email_new.css',
+          ]
+        }
+      }
+    },
+
+    uglify: {
+      bower: {
+        options: {
+          mangle: true,
+          compress: true,
+        },
+        files: {
+          'opwen_email_client/webapp/static/js/printThis.min.js': [
+            'node_modules/@bower_components/printThis/printThis.js',
+          ],
+          'opwen_email_client/webapp/static/js/jquery.lazyload.min.js': [
+            'node_modules/jquery-lazyload/jquery.lazyload.js'
+          ]
+        }
+      },
+      app: {
+        options: {
+          mangle: true,
+          compress: true,
+        },
+        files: {
+          'opwen_email_client/webapp/static/js/_base.min.js': [
+            'opwen_email_client/webapp/static/js/_base.js',
+          ],
+          'opwen_email_client/webapp/static/js/_base_email.min.js': [
+            'opwen_email_client/webapp/static/js/_base_email.js',
+          ],
+          'opwen_email_client/webapp/static/js/email_new.min.js': [
+            'opwen_email_client/webapp/static/js/email_new.js',
+          ],
+          'opwen_email_client/webapp/static/js/register.min.js': [
+            'opwen_email_client/webapp/static/js/register.js',
+          ],
+          'opwen_email_client/webapp/static/js/settings.min.js': [
+            'opwen_email_client/webapp/static/js/settings.js',
+          ]
+        }
+      }
+    }
+
+  });
+
+  grunt.registerTask('default', [
+    'copy:bower',
+    'cssmin',
+    'uglify:bower',
+    'uglify:app'
+  ])
+};

MANIFEST.in

@@ -0,0 +1,7 @@
+include requirements*.txt
+include README.rst
+include manage.py
+recursive-include opwen_email_client/webapp/static *
+recursive-include opwen_email_client/webapp/templates *.html
+recursive-include opwen_email_client/webapp/translations *.po
+recursive-include opwen_email_client/webapp/translations *.mo

README.rst

@@ -5,6 +5,9 @@ Opwen cloudserver
 .. image:: https://travis-ci.org/ascoderu/opwen-cloudserver.svg?branch=master
   :target: https://travis-ci.org/ascoderu/opwen-cloudserver
 
+.. image:: https://img.shields.io/pypi/v/opwen_email_client.svg
+  :target: https://pypi.python.org/pypi/opwen_email_client/
+
 .. image:: https://pyup.io/repos/github/ascoderu/opwen-cloudserver/shield.svg
   :target: https://pyup.io/repos/github/ascoderu/opwen-cloudserver/
 
@@ -15,19 +18,70 @@ Opwen cloudserver
 What's this?
 ------------
 
-This repository contains the source code for the Lokole cloud server. Its
-purpose is to connect the `application <https://github.com/ascoderu/opwen-webapp>`_
-running on the Lokole devices to the rest of the world. Lokole is a project
-by the Canadian-Congolese non-profit `Ascoderu <https://ascoderu.ca>`_.
+This repository contains the source code for the Lokole project by the
+Canadian-Congolese non-profit `Ascoderu <https://ascoderu.ca>`_. The Lokole
+project consists of two main parts: an email client and an email server.
+
+The Lokole email client is a simple application that offers functionality like:
+
+1. Self-service creation of user accounts
+2. Read emails sent to the account
+3. Write emails including rich formatting
+4. Send attachments
+
+All emails are stored in a local SQLite database. Once per day, the emails that
+were written during the past 24 hours get exported from the database, stored in
+a JSON file, compressed and uploaded to a location on Azure Blob Storage. The
+Lokole Server picks up these JSON files, manages the actual mailboxes for the
+users on the Lokole and sends new emails back to the Lokole by using the same
+compressed file exchange format.
 
-The server is implemented using `Connexion <https://jobs.zalando.com/tech/blog/crafting-effective-microservices-in-python/>`_
+The Lokole email application is intended to run on low-spec Raspberry Pi 3
+hardware (or similar). Read the "Production setup" section below for further
+information on how to set up the client devices.
+
+The Lokole email server is implemented using `Connexion <https://jobs.zalando.com/tech/blog/crafting-effective-microservices-in-python/>`_
 and has two main responsibilities:
 
 1. Receive emails from the internet that are addressed to Lokole users and
    forward them to the appropriate Lokole device.
 2. Send new emails created by Lokole users to the rest of the internet.
 
-More background information can be found in the `opwen-webapp README <https://github.com/ascoderu/opwen-webapp/blob/master/README.rst>`_.
+-------------------
+Why is this useful?
+-------------------
+
+Email is at the core of our modern life, letting us keep in touch with friends
+and family, connecting us to our businesses partners and fostering innovation
+through exchange of information.
+
+However, in many parts of the developing world, email access is not very
+wide-spread, usually because bandwidth costs are prohibitively high compared to
+local purchasing power. For example, in the Democratic Republic of the Congo
+(DRC) only 3% of the population have access to emails which leaves 75 million
+people unconnected.
+
+The Lokole is a project by the Canadian-Congolese non-profit `Ascoderu <https://ascoderu.ca>`_
+that aims to address this problem by tackling it from three perspectives:
+
+1. The Lokole is an email client that only uses bandwidth on a schedule. This
+   reduces the cost of service as bandwidth can now be purchased when the cost
+   is lowest. For example, in the DRC, $1 purchases only 65 MB of data during
+   peak hours. At night, however, the same amount of money buys 1 GB of data.
+
+2. The Lokole uses an efficient data exchange format plus compression so that
+   it uses minimal amounts of bandwidth, reducing the cost of service. All
+   expensive operations (e.g. creating and sending of emails with headers,
+   managing mailboxes, etc.) are performed on a server in a country where
+   bandwidth is cheap.
+
+3. The Lokole only uses bandwidth in batches. This means that the cost of
+   service can be spread over many people and higher savings from increased
+   compression ratios can be achieved. For example, individually purchasing
+   bandwidth for $1 to check emails is economically un-viable for most people
+   in the DRC. However, the same $1 can buy enough bandwidth to provide email
+   for hundreds of people via the Lokole. Spreading the cost in this way makes
+   email access sustainable for local communities.
 
 ---------------
 System overview
@@ -155,6 +209,7 @@ to initialize the required cloud resources.
   EOM
 
   cat > ${PWD}/secrets/users.env << EOM
+  OPWEN_SESSION_KEY={some secret for user session management}
   LOKOLE_REGISTRATION_USERNAME={some username for the registration endpoint}
   LOKOLE_REGISTRATION_PASSWORD={some password for the registration endpoint}
   EOM
@@ -196,4 +251,72 @@ The script will then provision a cluster in Azure Kubernetes Service and
 install the project via Helm. The secrets to connect to the provisioned
 cluster will be stored in the :code:`secrets` directory.
 
+There is a `script <https://github.com/ascoderu/opwen-cloudserver/blob/master/install.py>`_
+to set up a new Lokole email client. The script will install the email app in this
+repository as well as standard infrastructure like nginx and gunicorn.
+The script will also make ready peripherals like the USB modem used for data
+exchange, and set up any required background jobs such as the email
+synchronization cron job.
+
+The setup script assumes that you have already set up:
+
+* 3 Azure Storage Accounts, general purpose: for the cloudserver to manage its
+  queues, tables and blobs.
+* 1 Azure Storage Account, blob storage: for the cloudserver and email app to
+  exchange email packages.
+* 1 Application Insights account: to collect logs from the cloudserver and
+  monitor its operations.
+* 1 SendGrid account: to send and receive emails in the cloudserver.
+
+The setup script is tested with hardware:
+
+* `Raspberry Pi 3 <https://www.raspberrypi.org/products/raspberry-pi-3-model-b/>`_
+  running Raspbian Jessie lite
+  `v2016-05-27 <https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2016-05-31/2016-05-27-raspbian-jessie-lite.zip>`_,
+  `v2017-01-11 <https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2017-01-10/2017-01-11-raspbian-jessie-lite.zip>`_,
+  `v2017-04-10 <https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2017-04-10/2017-04-10-raspbian-jessie-lite.zip>`_, and
+  `v2017-11-29 <http://vx2-downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2017-12-01/2017-11-29-raspbian-stretch-lite.zip>`_.
+
+* `Orange Pi Zero <http://www.orangepi.org/orangepizero/>`_
+  running `Armbian Ubuntu Xenial <https://dl.armbian.com/orangepizero/Ubuntu_xenial_default.7z>`_
+
+The setup script is also tested with USB modems:
+
+* `Huawei E303s-65 <http://consumer.huawei.com/cl/mobile-broadband/dongles/tech-specs/e303-cl.htm>`_
+* `Huawei E3131 <http://consumer.huawei.com/lk/mobile-broadband/dongles/tech-specs/e3131-lk.htm>`_
+* `Huawei MS2131i-8 <http://consumer.huawei.com/en/solutions/m2m-solutions/products/tech-specs/ms2131-en.htm>`_
+
+The setup script installs the latest version of the email app published to PyPI.
+New versions get automatically published to PyPI (via Travis) whenever a new
+`release <https://github.com/ascoderu/opwen-cloudserver/releases/new>`_ is created
+on Github.
+
+You can run the script on your client device like so:
+
+.. sourcecode :: sh
+
+  curl -fsO https://raw.githubusercontent.com/ascoderu/opwen-cloudserver/master/install.py && \
+  sudo python3 install.py <client-name> <sim-type> <sync-schedule> <registration-credentials>
+
+---------------------
+Adding a new language
+---------------------
+
+To translate Lokole to a new language, install Python, `Babel <https://babel.pocoo.org/>`_
+and a translation editor such as `poedit <https://poedit.net/>`_. Then follow the steps below.
+
+.. sourcecode :: sh
+
+  # set this to the ISO 639-1 language code for which you are adding the translation
+  export language=ln
+
+  # generate the translation file
+  pybabel init -i babel.pot -d opwen_email_client/webapp/translations -l "${language}"
+
+  # fill-in the translation file
+  poedit "opwen_email_client/webapp/translations/${language}/LC_MESSAGES/messages.po"
+
+  # finalize the translation file
+  pybabel compile -d opwen_email_client/webapp/translations
+
 [ ~ Dependencies scanned by PyUp.io ~ ]

babel.cfg

@@ -0,0 +1,3 @@
+[python: **.py]
+[jinja2: **/templates/**.html]
+extensions=jinja2.ext.autoescape,jinja2.ext.with_

docker-compose.yml

@@ -79,11 +79,15 @@ services:
     build:
       context: .
       dockerfile: docker/client/Dockerfile
+      target: ${BUILD_TARGET}
       args:
-        LOKOLE_PORT: "8080"
-        LOKOLE_CLIENT_VERSION: ${WEBAPP_VERSION}
+        VERSION: ${BUILD_TAG}
     environment:
       <<: *shared-app-environment
+      OPWEN_EMAIL_SERVER_HOSTNAME: nginx:8888
+      WEBAPP_WORKERS: ${WEBAPP_WORKERS}
+      PORT: "8080"
+      HOST: "0.0.0.0"
     depends_on:
       - azurite