Dedupe emails on date and recipients (#206)

opwen_email_server/actions.py

@@ -24,6 +24,7 @@
 from opwen_email_server.utils.serialization import from_base64
 from opwen_email_server.utils.serialization import from_jsonl_bytes
 from opwen_email_server.utils.serialization import to_base64
+from opwen_email_server.utils.serialization import to_json
 from opwen_email_server.utils.serialization import to_jsonl_bytes
 from opwen_email_server.utils.string import is_lowercase
 
@@ -88,14 +89,15 @@ def _action(self, resource_id):  # type: ignore
             return 'skipped', 202
 
         email = self._email_parser(mime_email)
-        self._store_inbound_email(resource_id, email)
+        self._store_inbound_email(email)
 
         self._raw_email_storage.delete(resource_id)
 
         self.log_event(events.EMAIL_STORED_FOR_CLIENT, {'domain': get_domain(email.get('from') or '')})  # noqa: E501
         return 'OK', 200
 
-    def _store_inbound_email(self, email_id: str, email: dict):
+    def _store_inbound_email(self, email: dict):
+        email_id = self._to_id(email)
         email['_uid'] = email_id
 
         self._email_storage.store_object(email_id, email)
@@ -110,6 +112,10 @@ def _parse_mime_email(self, mime_email: str) -> dict:
         email = format_inline_images(email, self.log_warning)
         return email
 
+    @classmethod
+    def _to_id(cls, email: dict) -> str:
+        return sha256(to_json(email).encode('utf-8')).hexdigest()
+
 
 class StoreWrittenClientEmails(_Action):
     def __init__(self,

opwen_email_server/utils/email_parser.py

@@ -51,7 +51,7 @@ def _parse_attachments(mailparts: Iterable[MailPart]) -> Iterable[dict]:
 
 
 def _parse_addresses(message: PyzMessage, address_type: str) -> List[str]:
-    return [email for _, email in message.get_addresses(address_type) if email]
+    return sorted(email for _, email in message.get_addresses(address_type) if email)
 
 
 def _parse_address(message: PyzMessage, address_type: str) -> Optional[str]:

tests/opwen_email_server/test_actions.py

@@ -107,23 +107,26 @@ def throw(*args, **kwargs):
         self.assertFalse(self.email_parser.called)
 
     def test_200(self):
-        resource_id = str(uuid4())
+        resource_id = 'b8dcaf40-fd14-4a89-8898-c9514b0ad724'
         domain = 'test.com'
         raw_email = 'dummy-mime'
-        email = {'to': ['foo@{}'.format(domain)], '_uid': resource_id}
+        parsed_email = {'to': ['foo@{}'.format(domain)]}
+        email_id = 'eaee7d51a5440981d5df66825134dcfe0490f145e1ad689bce8c68caeaecde53'
+        stored_email = dict(parsed_email)
+        stored_email['_uid'] = email_id
 
         self.raw_email_storage.fetch_text.return_value = raw_email
         self.pending_factory.return_value = self.pending_storage
-        self.email_parser.return_value = email
+        self.email_parser.return_value = parsed_email
 
         _, status = self._execute_action(resource_id)
 
         self.assertEqual(status, 200)
         self.raw_email_storage.fetch_text.assert_called_once_with(resource_id)
         self.raw_email_storage.delete.assert_called_once_with(resource_id)
-        self.email_storage.store_object.assert_called_once_with(resource_id, email)
+        self.email_storage.store_object.assert_called_once_with(email_id, stored_email)
         self.pending_factory.assert_called_once_with(domain)
-        self.pending_storage.store_text.assert_called_once_with(resource_id, 'pending')
+        self.pending_storage.store_text.assert_called_once_with(email_id, 'pending')
         self.email_parser.assert_called_once_with(raw_email)
 
     def _execute_action(self, *args, **kwargs):

tests/opwen_email_server/utils/test_email_parser.py

@@ -54,8 +54,8 @@ def test_parses_email_with_cc_and_bcc(self):
         email = email_parser.parse_mime_email(mime_email)
 
         self.assertEqual(email.get('bcc'), ['laura@lokole.ca'])
-        self.assertEqual(email.get('cc'), ['nzola@lokole.ca',
-                                           'clemens@lokole.ca'])
+        self.assertEqual(email.get('cc'), ['clemens@lokole.ca',
+                                           'nzola@lokole.ca'])
 
     def test_parses_email_with_attachments(self):
         mime_email = self._given_mime_email('email-attachment.mime')