Make webmail user management case insensitive (#423)

* Make webmail user management case insensitive

* Enable versioning of client emails test package

* Increase integration test delay for live tests

* Visualize test delays

.env

@@ -15,6 +15,7 @@ LOKOLE_EMAIL_SERVER_QUEUES_NAMESPACE=
 LOKOLE_SENDGRID_KEY=
 LOKOLE_RESOURCE_SUFFIX=
 REGISTRATION_CREDENTIALS=admin:password
+TEST_STEP_DELAY=10
 
 CLOUDBROWSER_PORT=10001
 AZURITE_PORT=10000

.travis.yml

@@ -26,6 +26,7 @@ before_script: |
     export AZURITE_KEY="$TEST_AZURE_STORAGE_KEY"
     export AZURITE_HOST=""
     export AZURITE_SECURE="True"
+    export TEST_STEP_DELAY=90
   else
     export REGISTRATION_CREDENTIALS="admin:password"
     export LOKOLE_QUEUE_BROKER_SCHEME="amqp"

docker/docker-compose.test.yml

@@ -13,5 +13,6 @@ services:
       AZURITE_ACCOUNT: ${AZURITE_ACCOUNT}
       AZURITE_KEY: ${AZURITE_KEY}
       AZURITE_HOST: ${AZURITE_HOST}
+      TEST_STEP_DELAY: ${TEST_STEP_DELAY}
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock

docker/integtest/2-client-uploads-emails.sh

@@ -9,6 +9,8 @@ mkdir -p "${out_dir}"
 . "${scriptdir}/utils.sh"
 
 emails_to_send="${in_dir}/client-emails.tar.gz"
+tar -czf "${emails_to_send}" -C "${in_dir}" emails.jsonl zzusers.jsonl
+
 client_id="$(jq -r '.client_id' < "${out_dir}/register1.json")"
 resource_container="$(jq -r '.resource_container' < "${out_dir}/register1.json")"
 resource_id="$(uuidgen).tar.gz"

docker/integtest/3-receive-email-for-client.sh

@@ -26,3 +26,10 @@ http --check-status -f POST \
   "dkim={@sendgrid.com : pass}" \
   "SPF=pass" \
   "email=@${email_to_receive}"
+
+# simulate delivery of another email
+http --check-status -f POST \
+  "http://nginx:8888/api/email/sendgrid/${client_id}" \
+  "dkim={@sendgrid.com : pass}" \
+  "SPF=pass" \
+  "email=@${in_dir}/inbound-email-2.mime"

docker/integtest/4-client-downloads-emails.sh

@@ -7,6 +7,10 @@ mkdir -p "${out_dir}"
 # shellcheck disable=SC1090
 . "${scriptdir}/utils.sh"
 
+declare -A num_emails_expected_for_client
+num_emails_expected_for_client[1]=2
+num_emails_expected_for_client[2]=1
+
 for i in 1 2; do
 
 client_id="$(jq -r '.client_id' < "${out_dir}/register${i}.json")"
@@ -29,7 +33,7 @@ az_storage download "${resource_container}" "${resource_id}" "${out_dir}/downloa
 tar xzf "${out_dir}/downloaded${i}.tar.gz" -C "${out_dir}"
 
 num_emails_actual="$(wc -l "${out_dir}/emails.jsonl" | cut -d' ' -f1)"
-num_emails_expected=1
+num_emails_expected="${num_emails_expected_for_client[${i}]}"
 
 if [[ "${num_emails_actual}" -ne "${num_emails_expected}" ]]; then
   echo "Got ${num_emails_actual} emails but expected ${num_emails_expected}" >&2

docker/integtest/files/.gitignore

@@ -1 +1,2 @@
 test.out/
+client-emails.tar.gz

docker/integtest/files/emails.jsonl

@@ -0,0 +1,3 @@
+{"from":"clemens@developer1.lokole.ca","to":["clemens.wolff@gmail.com","laura.barluzzi@gmail.com"],"subject":"First test email sent from Lokole client","body":"Some content","_uid":"ed262575-73db-49cc-9c87-e2e95f8ac9cc","sent_at":"2019-10-26 11:23"}
+{"from":"laura@developer1.lokole.ca","to":["clemens.wolff@gmail.com"],"subject":"Second test email sent from Lokole client","body":"Some more content","_uid":"d03fff22-5c51-4f10-8111-5b24686964ba","attachments":[{"filename":"test.txt","content":"dGVzdCBhdHRhY2htZW50"}],"sent_at":"2019-10-25 05:00"}
+{"from":"Clemens@developer1.lokole.ca","to":["clemens@gmail.com","laura.barluzzi@gmail.com"],"subject":"Third test email sent from Lokole client","body":"Some extra content","_uid":"43f619b3-64ba-4ea3-963d-740cf0e58989","sent_at":"2019-10-26 12:34"}

docker/integtest/files/inbound-email-2.mime

@@ -0,0 +1,27 @@
+Received: by mx0028p1mdw1.sendgrid.net with SMTP id Yt3NEnbnLU Mon, 13 Feb 2017 06:25:41 +0000 (UTC)
+Received: from mail-yw0-f176.google.com (mail-yw0-f176.google.com [209.85.161.176]) by mx0028p1mdw1.sendgrid.net (Postfix) with ESMTPS id C726D640B63; Mon, 13 Feb 2017 06:25:41 +0000 (UTC)
+Received: by mail-yw0-f176.google.com with SMTP id w75so45612320ywg.1; Sun, 12 Feb 2017 22:25:41 -0800 (PST)
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=mime-version:from:date:message-id:subject:to; bh=ViHLGS6kOdo9Q9CkDDSSSS3bgKuN0a+UXhwMw06ak4Q=; b=f3WGzjgLe0tPG2edhiHxiCEZatThUga/qJFnWZNyY4lEVjRM9l3qn1BZ4ITawT9tDK LS6qFx//6in7u0rV0YKoa8TfScUFOpPHGCmq1Wxdp7mrWP7GDuCOz3LzyXQsrBe/erGy YEjAVU876sWJ109mcMcmbgOL1SD3d4ak+8GVBSC8oMKPj5XWZsET7WmsonhKf5PHE9IW eJHKqdOkxiPbmDutVx7uS1Bi5u4d9UYPhgxFwAK9lWyJ/Esw6yffjlrUvmQCPibSCxRv o979yY6FyJXDJ82l4ErntcOloFNpzWZ89WkRhb1aBLUoZs3402s6D3wC0ljpmvneIAkw 3D6A==
+X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:mime-version:from:date:message-id:subject:to; bh=ViHLGS6kOdo9Q9CkDDSSSS3bgKuN0a+UXhwMw06ak4Q=; b=sWY7uU6kK3dg62wVuxcLsRYLg3eGcoLuoLjL0Ju/sl9rGqSDxVc2saIS0ThfUaHlfZ g1zvF+rBoxa7v9jk7MhEw3izW01WXDMm0w2JGc1QLTo3ZM2xW9Clss63R3ZtNKabuyhd 77NHAgbarmQGW5XuqwS1Fy0NMWHkAlLsZd2AnkNb6gCI/VHCCv/oem19bWvNWwRTPBYE cQDPJfzRiUzRPNZPLtlL5ybd2yyb4lcuG+2QoQV8uxPsKS4eDOjNmM76UWZ9s/Ul/mR+ Qbyui7suOO0vPy8GFJHPV9X2ffLqesafTAetCj3LClCdLIdfQDaK86mmVHOT6zldeCTa HH6Q==
+X-Gm-Message-State: AMke39n2h/OZU6fwgOdDltzsKqISVbe3ez6t19OeVrg2sT3pDRhSSQiIcwGzKjdWOD/oX96rQlTi0O9t9yhUfA==
+X-Received: by 10.129.81.4 with SMTP id f4mr15409224ywb.239.1486967141412; Sun, 12 Feb 2017 22:25:41 -0800 (PST)
+MIME-Version: 1.0
+Received: by 10.129.156.139 with HTTP; Sun, 12 Feb 2017 22:25:01 -0800 (PST)
+From: Clemens Wolff <clemens.wolff@gmail.com>
+Date: Sun, 12 Feb 2017 22:25:01 -0800
+Message-ID: <CAL79TcnjhV5PinZVY8Y3QEoNNcSa9uuNU5N3EP-gqcYPFfuHLA@mail.gmail.com>
+Subject: Two recipients
+To: Clemens@developer1.lokole.ca, LaUrA@developer1.lokole.ca
+Content-Type: multipart/alternative; boundary=001a1146392641b94705486384bf
+
+--001a1146392641b94705486384bf
+Content-Type: text/plain; charset=UTF-8
+
+Body of the message.
+
+--001a1146392641b94705486384bf
+Content-Type: text/html; charset=UTF-8
+
+<div dir="ltr">Body of the message.</div>
+
+--001a1146392641b94705486384bf--

docker/integtest/files/zzusers.jsonl

@@ -0,0 +1 @@
+{"email":"clemens@developer1.lokole.ca","password":"$2b$12$9LaXqZMPJi0PiTY.95dIQOvc8LkYQzRlg5a9pDWX47L/npaYqynU2"}

docker/integtest/tests.sh

@@ -2,11 +2,13 @@
 set -eo pipefail
 
 scriptdir="$(dirname "$0")"
+# shellcheck disable=SC1090
+. "${scriptdir}/utils.sh"
 
 "${scriptdir}/0-wait-for-services.sh"
 "${scriptdir}/1-register-client.sh"
-"${scriptdir}/2-client-uploads-emails.sh" && sleep 10s
-"${scriptdir}/3-receive-email-for-client.sh" && sleep 10s
+"${scriptdir}/2-client-uploads-emails.sh" && wait_seconds "${TEST_STEP_DELAY}"
+"${scriptdir}/3-receive-email-for-client.sh" && wait_seconds "${TEST_STEP_DELAY}"
 "${scriptdir}/4-client-downloads-emails.sh"
 "${scriptdir}/5-assert-on-results.sh"
 

docker/integtest/utils.sh

@@ -30,3 +30,15 @@ az_storage() {
     --connection-string "$(az_connection_string)" \
   > /dev/null
 }
+
+wait_seconds() {
+  local seconds="$1"
+
+  printf 'Waiting' >&2
+  while [[ "${seconds}" -gt 0 ]]; do
+    printf '.' >&2
+    sleep 1
+    seconds="$((seconds - 1))"
+  done
+  echo >&2
+}

opwen_email_client/webapp/templates/macros/email.html

@@ -27,7 +27,7 @@ <h4 class="panel-title">
               {% if show_sent_to %}
               <div class="col-sm-3">
                 <span class="email-sent-to">
-                  {% if email['from'] == current_user.email %}
+                  {% if email['from'].lower() == current_user.email.lower() %}
                     <span class="visible-print">{{ _('To: %(emails)s', emails=', '.join(email['to'])) }}</span>
                     <span class="hidden-print">{{ email['to'] | join(', ') }}</span>
                   {% else %}

opwen_email_server/integration/azure.py

@@ -69,6 +69,7 @@ def get_user_storage() -> AzureObjectStorage:
         secure=config.TABLES_SECURE,
         container=config.CONTAINER_USERS,
         provider=config.STORAGE_PROVIDER,
+        case_sensitive=False,
     )
 
 
@@ -81,6 +82,7 @@ def get_mailbox_storage() -> AzureTextStorage:
         secure=config.BLOBS_SECURE,
         container=config.CONTAINER_MAILBOX,
         provider=config.STORAGE_PROVIDER,
+        case_sensitive=False,
     )
 
 

opwen_email_server/services/storage.py

@@ -11,6 +11,7 @@
 
 from cached_property import cached_property
 from libcloud.storage.base import Container
+from libcloud.storage.base import Object
 from libcloud.storage.base import StorageDriver
 from libcloud.storage.providers import get_driver
 from libcloud.storage.types import ContainerAlreadyExistsError
@@ -35,36 +36,73 @@
 Download = Tuple[str, Callable[[bytes], dict]]
 
 
+class _Container:
+    def __init__(self, wrapped: Container):
+        self._wrapped = wrapped
+
+    def get_object(self, object_name: str) -> Object:
+        return self._wrapped.get_object(object_name)
+
+    def iterate_objects(self, prefix: Optional[str] = None) -> Iterable[Object]:
+        return self._wrapped.iterate_objects(prefix)
+
+    def upload_object(self, file_path: str, object_name: str) -> Object:
+        return self._wrapped.upload_object(file_path, object_name)
+
+    def upload_object_via_stream(self, iterator: Iterator[bytes], object_name: str) -> Object:
+        return self._wrapped.upload_object_via_stream(iterator, object_name)
+
+
+class _CaseInsensitiveContainer(_Container):
+    def get_object(self, object_name: str) -> Object:
+        object_name = object_name.lower()
+        return super().get_object(object_name)
+
+    def iterate_objects(self, prefix: Optional[str] = None) -> Iterable[Object]:
+        prefix = prefix.lower() if prefix is not None else None
+        return super().iterate_objects(prefix)
+
+    def upload_object(self, file_path: str, object_name: str) -> Object:
+        object_name = object_name.lower()
+        return super().upload_object(file_path, object_name)
+
+    def upload_object_via_stream(self, iterator: Iterator[bytes], object_name: str) -> Object:
+        object_name = object_name.lower()
+        return super().upload_object_via_stream(iterator, object_name)
+
+
 class _BaseAzureStorage(LogMixin):
     def __init__(self,
                  account: str,
                  key: str,
                  container: str,
                  provider: str,
                  host: Optional[str] = None,
-                 secure: bool = True) -> None:
+                 secure: bool = True,
+                 case_sensitive: bool = True) -> None:
         self._account = account
         self._key = key
         self._container = container
         self._provider = getattr(Provider, provider)
         self._host = host or None
         self._secure = secure
+        self._case_sensitive = case_sensitive
 
     @cached_property
     def _driver(self) -> StorageDriver:
         driver = get_driver(self._provider)
         return driver(self._account, self._key, host=self._host, secure=self._secure)
 
     @cached_property
-    def _client(self) -> Container:
+    def _client(self) -> _Container:
         try:
             container = self._driver.get_container(self._container)
         except ContainerDoesNotExistError:
             try:
                 container = self._driver.create_container(self._container)
             except ContainerAlreadyExistsError:
                 container = self._driver.get_container(self._container)
-        return container
+        return _Container(container) if self._case_sensitive else _CaseInsensitiveContainer(container)
 
     @property
     def _generated_suffix(self) -> str:
@@ -91,7 +129,7 @@ def delete(self, resource_id: str):
             self.log_debug('deleted %s', resource_id)
 
     def iter(self, prefix: Optional[str] = None) -> Iterator[str]:
-        resources = self._driver.iterate_container_objects(self._client, prefix=prefix)
+        resources = self._client.iterate_objects(prefix=prefix)
 
         for resource in resources:
             resource_id = resource.name

tests/opwen_email_server/services/test_storage.py

@@ -82,7 +82,7 @@ def get_container(*args, **kwargs):
             driver.get_container.side_effect = get_container
             driver.create_container.side_effect = throw(ContainerAlreadyExistsError(None, driver, self._container))
 
-            self.assertIs(self._storage._client, container)
+            self.assertIs(self._storage._client._wrapped, container)
 
     def setUp(self):
         self._folder = mkdtemp()
@@ -98,6 +98,42 @@ def tearDown(self):
         rmtree(self._folder)
 
 
+class AzureTextStorageCaseInsensitiveTests(TestCase):
+    def test_stores_fetches_and_deletes_text(self):
+        self._storage.store_text('iD1', 'some content')
+        actual_content = self._storage.fetch_text('Id1')
+
+        self.assertEqual(actual_content, 'some content')
+
+        self._storage.delete('id1')
+        with self.assertRaises(ObjectDoesNotExistError):
+            self._storage.fetch_text('id1')
+
+    def test_list_with_prefix(self):
+        self._storage.store_text('One/a', 'a')
+        self._storage.store_text('one/b.txt.gz', 'b')
+        self._storage.store_text('tWo/c.txt.gz', 'c')
+        self._storage.store_text('twO/d', 'd')
+        self._storage.store_text('two/e', 'e')
+        self._storage.store_text('f', 'f')
+        self.assertEqual(sorted(self._storage.iter('one/')), sorted(['a', 'b']))
+        self.assertEqual(sorted(self._storage.iter('two/')), sorted(['c', 'd', 'e']))
+
+    def setUp(self):
+        self._folder = mkdtemp()
+        self._container = 'container'
+        self._storage = AzureTextStorage(
+            account=self._folder,
+            key='key',
+            container=self._container,
+            provider='LOCAL',
+            case_sensitive=False,
+        )
+
+    def tearDown(self):
+        rmtree(self._folder)
+
+
 class AzureFileStorageTests(TestCase):
     def test_stores_fetches_and_deletes_file(self):
         resource_id, expected_content = 'id1', 'some content'