Add Stalwart 0.16 JMAP admin compatibility (#628, #1168) (#1131)

* Split up MailClient from clients.py into 3 different MailClient (incl interface.)

* Some initial jmap work

Not happy with the current state, but also not happy with existing jmap
clients. The idea is to make this pretty lean but still allow for some
developer ux wins.

* Kick-off stalwart types, and start with get domain

* WIP integration of pydantic

* Fix jmap calls with, and add get_account

* Add a few more types, wrangle `@type`, and add a few more methods to stalwart client jmap.

* Base types off of BaseSchema and adjust properties to snake_case because python

* Finish up create_account

* Refactor clients test structure

* Fix create_account's return, quota param, and deprecate app password. (It's not used.)

* Update docblock for mail_client_jmap.py

* Cache primary domain id, and fix get_account / add delete_account

* * Move types into `mail.types.*` instead of being *_types.py inside client.
* Rename AccountType/DomainType to just Account/Domain.

* Include test script

* * Add update_individual
* Port jmap types to pydantic

* Add save_app_password

* Refactor how the jmap classes are setup so we don't confused admin calls with user calls.

* Move domain lookup to its own function and replace the instance in create_account

* WIP refactor to split up basic jmap ops

* Refactor jmap clients, and fix up save_email_addresses

* Refactor invocation names to the constants

* Implement replace/delete email addresses, fix delete account, and swap exclude_none to exclude_unset

* Add StorageQuota type, and implement update_quota

* Refactor and add DkimSignature type

* Move JMapType and Id into jmap types, and add get_identity with types.

* Create private key and implement create_dkim for dkim1 signatures

* Remove key dumper test code

* Add delete_dkim

* Implement most of the dkim/dns functions, and modify some return types.

* Re-add function I accidentally cut

* Merge stalwart v0.16 into main docker file so I can start running tests.

Config is stored inside sqlite file, hopefully this is cross-compat...

* Swap listener to 8180 for http

* Add some minor hacks to get the dashboard working

* Get domain verification working-ish for migrated users

* Move main.db to main.db.example

* Move config.json to config.json.example

* Only volume mount the db file not the folder

Otherwise we'll run into "attempt to write a readonly database" error.

* Update bootstrap.py for v0.16

* Lint

* Format

* Fix import

* Foce test_legacy to use mailclient legacy

* `chmod 777 main.db` which gets around the cannot read readonly db error.

* Hook-up new stalwart url, user, pass, and auth method

* Add response dump files, an initial test and blank out debug dump

* Lint / Format

* Remove unused import

* Fix ruff format checks

---------

Co-authored-by: Davi Nakano <davinakanoca@gmail.com>

Author: 2 people

.env.example

@@ -98,6 +98,13 @@ STALWART_API_AUTH_METHOD=basic
 # Enable after Stalwart 0.16 upgrade
 STALWART_DKIM_STAGE_MANAGEMENT_ENABLED=false
 
+# Use the v0.16 jmap api instead of the pre-v0.16 rest based api
+STALWART_ADMIN_API_USE_JMAP=false
+STALWART_JMAP_API_URL=http://stalwart_new:8080
+STALWART_JMAP_API_AUTH_USER=admin
+STALWART_JMAP_API_AUTH_SECRET=admin
+STALWART_JMAP_API_AUTH_METHOD=basic
+
 # Hosted DKIM DNS delegation. Customers publish CNAMEs under their own domains
 # to these hosted records so Thunderbird can rotate DKIM keys later.
 HOSTED_DKIM_DOMAIN=dkim.thunderhosted.com

.env.test

@@ -87,6 +87,13 @@ STALWART_BASE_API_URL=http://stalwart:8080
 STALWART_API_KEY=this-is-mocked
 STALWART_API_AUTH_METHOD=basic
 
+# Use the v0.16 jmap api instead of the pre-v0.16 rest based api
+STALWART_ADMIN_API_USE_JMAP=false
+STALWART_JMAP_API_URL=http://stalwart_new:8080
+STALWART_JMAP_API_AUTH_USER=admin
+STALWART_JMAP_API_AUTH_SECRET=admin
+STALWART_JMAP_API_AUTH_METHOD=basic
+
 # Keycloak's (or another oidc provider) client settings
 OIDC_CLIENT_ID=tb-accounts
 OIDC_CLIENT_SECRET=

bootstrap.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 import argparse
 import os
 import shutil
@@ -10,35 +11,70 @@
     action='store_true',
     help='deletes any existing configs. WARNING: Make sure to backup your local env, and stalwart configs!',
 )
+arg_parse.add_argument(
+    '--v016-only',
+    action='store_true',
+    help='Only bootstraps Stalwart v0.16 / Stalwart New',
+)
 
 args = arg_parse.parse_args()
 
 
-def main():
-    from_scratch = args.from_scratch
-
-    print('Bootstrapping project:')  # noqa: T201
+def bootstrap_legacy(from_scratch):
     if not os.path.isfile('.env') or from_scratch:
         try:
             shutil.copy('.env.example', '.env')
             print('\t* Copied .env.example to .env')  # noqa: T201
         except SameFileError:
             pass
 
-    if os.path.isdir('mail/data') and from_scratch:
-        shutil.rmtree('mail/data')
-        print('\t* Removed mail/data')  # noqa: T201
+    if os.path.isdir('mail/stalwart_legacy/data') and from_scratch:
+        shutil.rmtree('mail/stalwart_legacy/data')
+        print('\t* Removed mail/stalwart_legacy/data')  # noqa: T201
 
-    if not os.path.isfile('mail/etc/config.toml') or from_scratch:
+    if not os.path.isfile('mail/stalwart_legacy/etc/config.toml') or from_scratch:
         try:
-            os.makedirs('mail/etc', exist_ok=True)
-            shutil.copy('config.toml.example', 'mail/etc/config.toml')
-            print('\t* Copied config.toml.example to mail/etc/config.toml')  # noqa: T201
+            os.makedirs('mail/stalwart_legacy/etc/', exist_ok=True)
+            shutil.copy('config.toml.example', 'mail/stalwart_legacy/etc/config.toml')
+            print('\t* Copied config.toml.example to mail/stalwart_legacy/etc/config.toml')  # noqa: T201
         except SameFileError:
             pass
 
+
+def bootstrap_new(from_scratch):
+    # Handle v0.16
+    path = Path('mail/stalwart/lib')
+    if os.path.isdir(path) and from_scratch:
+        os.remove(path / 'main.db')
+        print(f'\t* Removed {path}/main.db')  # noqa: T201
+    if not os.path.isfile(path / 'main.db'):
+        shutil.copy(path / 'main.db.example', path / 'main.db')
+        print(f'\t* Copied {path}/main.db.example to {path}/main.db')  # noqa: T201
+
+    path = Path('mail/stalwart/etc')
+    if os.path.isdir(path) and from_scratch:
+        os.remove(path / 'config.json')
+        print(f'\t* Removed {path}/config.json')  # noqa: T201
+    if not os.path.isfile(path / 'config.json'):
+        shutil.copy(path / 'config.json.example', path / 'config.json')
+        print(f'\t* Copied {path}/config.json.example to {path}/config.json')  # noqa: T201
+
     print('Finished!')  # noqa: T201
 
 
+def main():
+    new_only = args.v016_only
+    from_scratch = args.from_scratch
+
+    print('Bootstrapping project:', args.__dict__)  # noqa: T201
+
+    if not new_only:
+        print('Bootstrapping v0.15')  # noqa: T201
+        bootstrap_legacy(from_scratch)
+
+    print('Bootstrapping v0.16')  # noqa: T201
+    bootstrap_new(from_scratch)
+
+
 if __name__ == '__main__':
     main()

docker-compose.yml

@@ -117,13 +117,13 @@ services:
       - "./templates:/app/templates:z"
       - "./keycloak/themes:/app/keycloak/themes:z"
 
-  stalwart:
+  stalwart_legacy:
     image: stalwartlabs/stalwart:v0.15.4
     #build:
     #  context: ../stalwart
     #  dockerfile: Dockerfile
     volumes:
-      - "./mail:/opt/stalwart:z"
+      - "./mail/stalwart_legacy:/opt/stalwart:z"
     ports:
       - "443:443"
       - "8080:8080"
@@ -141,6 +141,35 @@ services:
       ADMIN_SECRET: "{plain}accounts"
       MAIL_STALWART_PATH: "/opt/stalwart"
 
+  stalwart_new:
+    image: stalwartlabs/stalwart:v0.16
+    hostname: 'stalwart_new'
+    domainname: 'stalwart_new.local'
+    #entrypoint: bash
+    volumes:
+      - "./mail/stalwart/etc:/etc/stalwart:Z"
+      - "./mail/stalwart/lib/main.db:/var/lib/stalwart/main.db:Z"
+    ports:
+      - "8443:443"
+      - "8180:8080"
+    #  - "8081:8081"
+    #  - "25:25"
+    #  - "587:587"
+    #  - "465:465"
+    #  - "143:143"
+    #  - "993:993"
+    #  - "4190:4190"
+    #  - "110:110"
+    #  - "995:995"
+    environment:
+      STALWART_RECOVERY_ADMIN: "admin:admin" 
+      # admin@example.com : iKeBFzrCbRRzaAZW
+    networks:
+      default:
+        aliases:
+          - stalwart_new.local
+
+
   keycloak:
     #image: quay.io/keycloak/keycloak:latest
     build:

jmap_test.py

@@ -0,0 +1,30 @@
+"""MailClientJMAP test script. These examples are tested against a stand-alone Stalwart v0.16 instance.
+The results and responses will be used in unit tests."""
+
+from thunderbird_accounts.mail.clients.mail_client_jmap import MailClientAdminJMAP
+
+jwt = None
+client = MailClientAdminJMAP()
+# user_client = MailClientUserJMAP(username='pizza666@example.org', user_jwt=jwt)
+
+
+# print('account->', client.get_account('admin@example.org'))
+# print('domain->', client.get_domain('example.com'))
+print('set account->', client.create_account([], 'pizza@example.org', 'PIZZA!', quota=15_000))
+print('delete account->', client.delete_account('pizza@example.org'))
+# print('save email->', client.save_email_addresses('admin@example.org', ['lols2@example.com']))
+# print('create domain->', client.create_domain('pizza.lol', 'weeee'))
+# print('delete domain->', client.delete_domain('pizza.lol'))
+# app_password = user_client.save_app_password('my cool device!!')
+# print('create app password->', app_password)
+# print('delete app password->', user_client.delete_app_password(app_password.id))
+# print('save alias->', client.save_email_addresses('admin@example.org', ['pants123@example.org']))
+# print('remove alias->', client.delete_email_addresses('admin@example.org', ['pants123@example.org']))
+# print('???', user_client.get_identity())
+# print('create dkim->',client.create_dkim('example.org'))
+
+print('get dns record->', client._get_dns_records('example.org'))
+print('build dns record->', client.build_expected_dns_records('example.org'))
+print('check dns record->', client.check_domain_dns('faviconfetcher.ca'))
+
+# print('->', AccountUpdate(aliases={'0': EmailAlias(enabled=True, name='beans', domain_id='g')}).model_dump())

mail/.gitignore

@@ -1,3 +1,4 @@
-etc/config.toml
-data/*
-logs/*
+stalwart_legacy/data/*
+stalwart_legacy/logs/*
+stalwart/lib/main.db
+stalwart/etc/config.json

mail/stalwart/dump/d__handle_destroy.json

@@ -0,0 +1,22 @@
+{
+  "accountId": "d333333",
+  "notDestroyed": {
+    "d": {
+      "type": "objectIsLinked",
+      "objectId": {
+        "object": "Domain",
+        "id": "d"
+      },
+      "linkedObjects": [
+        {
+          "object": "DkimSignature",
+          "id": "i0eilb9iaaqb"
+        },
+        {
+          "object": "DkimSignature",
+          "id": "i0eilcgcabab"
+        }
+      ]
+    }
+  }
+}

mail/stalwart/dump/d_create_dkim.json

@@ -0,0 +1,8 @@
+{
+  "accountId": "d333333",
+  "created": {
+    "73f16c71-cb4b-42e4-85e7-ddb0f35a6444": {
+      "id": "i3cjmrt2acac"
+    }
+  }
+}

mail/stalwart/dump/d_create_domain.json

@@ -0,0 +1,8 @@
+{
+  "accountId": "d333333",
+  "created": {
+    "5e30f478-0934-493a-8639-d515053f475e": {
+      "id": "g"
+    }
+  }
+}

mail/stalwart/dump/d_delete_account.json

@@ -0,0 +1,3 @@
+[
+  "bv"
+]