Merge pull request #180 from SamSchott/develop

Release 1.2.0

.github/workflows/lint.yml

@@ -0,0 +1,40 @@
+name: Lint with flake8 and mypy
+
+on: [pull_request]
+
+jobs:
+  flake:
+    name: Lint with flake8
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.6
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.6
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8
+        pip install .
+    - name: Lint with flake8
+      run: |
+        flake8 . --count --ignore=C901,W503 --max-line-length=130 --statistics
+
+  mypy:
+    name: Type checking with mypy
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.6
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.6
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade mypy
+        pip install .
+    - name: Type checking with mypy
+      run: |
+        mypy maestral

.github/workflows/pythonpublish.yml → .github/workflows/publish.yml

@@ -1,4 +1,4 @@
-name: Upload Python Package
+name: Upload Python package to PyPI
 
 on:
   push:

.github/workflows/ci.yml → .github/workflows/test.yml

@@ -1,26 +1,8 @@
-name: CI
+name: Run tests
 
-on: [pull_request]
+on: pull_request
 
 jobs:
-  flake:
-    name: Lint with flake8
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.6
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.6
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install flake8
-        pip install .
-    - name: Lint with flake8
-      run: |
-        flake8 . --count --ignore=C901,W503 --max-line-length=127 --statistics
-
   tests:
     name: Run tests
     strategy:

.gitignore

@@ -88,6 +88,7 @@ celerybeat-schedule
 .venv
 env*
 venv*
+*venv
 ENV/
 env.bak/
 venv.bak/

CHANGELOG.md

@@ -1,3 +1,85 @@
+## v1.2.0
+
+The local file index and sync history are now stored in a SQLite database. After the
+update, Maestral will first reindex your Dropbox to populate the new index.
+
+This change enables several improvements to the command line interface and GUI: The
+command `maestral activity` now shows the progress of individual uploads or downloads.
+`maestral history` has been added to list recent sync events. In the GUI, the recent
+changes menu now has been replaced by a "Activity" window which shows all sync events of
+the past week.
+
+This release also introduces clickable desktop notifications, performance improvements to
+indexing local file changes, and bug fixes and smaller changes listed below.
+
+Finally, this is release introduces support for macOS 11 (Big Sur).
+
+#### Added:
+
+- Added an option `--external` to `maestral log show` to open the log in the platform's
+  default program instead of showing it in the console.
+- Added a CLI command `history` to show all sync events of the past week.
+- Added a "Activity" window to show all sync events of the past week.
+- Desktop notifications are now clickable: for a single file change, clicking the
+  notification will show the file in the platform's file manager. For a deletion, the
+  Dropbox website is opened to provide options for restoring the file or folder.
+- Use entry points to discover GUI frontends. 3rd party GUIs can register a `maestral_gui`
+  entry point to be launched with the `maestral gui` CLI command. If installed,
+  `maestral gui` will default to the 1st party `maestral-cocoa` or `maestral-qt` GUIs on
+  macOS and Linux, respectively.
+
+#### Changed:
+
+- Transition to short-lived auth tokens for newly linked accounts.
+- Transition to OAuth scopes for app permissions.
+- Save all sync history and local index in SQLite database.
+- Reduce unnecessary path conversions during indexing of local changes.
+- Improved performance on case-sensitive file systems.
+- Sync remote changes in filename even if they are only a change in casing. Those changes
+  where previously ignored.
+- Attempt to preserve local file permissions when syncing unless the file id has changed.
+  Dropbox servers do store file permissions but don't make them available through the
+  public API. We therefore cannot sync file permissions and instead choose not to
+  overwrite locally set permissions on every download.
+- Changed return type of `Maestral.get_activity` from namedtuple to dict for better
+  consistency throughout the API. Every uploading or downloading item will have 'size'
+  and 'completed' entries to monitor the progress of syncing individual items.
+- The CLI command `maestral activity` now shows the progress of uploads and downloads for
+  individual files.
+- Introduced type annotations throughout and fixed a few type-related bugs.
+- Added a field "Sync threads" to the output of the CLI command `maestral status`.
+- The output of `maestral ls` is now printed in a grid, similar to the `ls` command
+  included with most platforms.
+- The macOS app bundle now uses Python 3.8, leading to some performance improvements when
+  moving or copying file system trees.
+- Prepared the GUI for changes in macOS Big Sur: use native alerts and dialogs wherever
+  possible and refactor loading of libraries.
+- Use an asyncio event loop instead of Pyro's event loop to run the daemon. This enables
+  integration with the Cocoa run loop and callbacks when clicking notifications.
+
+#### Fixed:
+
+- Fixes a bug where throttling of sync threads would raise an error when we cannot
+  determine the CPU count.
+- Fixes a bug where sending SIGTERM to the daemon process would raise an error when we
+  cannot determine its PID. Now, `Stop.Failed` is returned instead.
+- Fixes a bug which would result in incorrect systemd unit files for non-default config
+  file names. Please disable and re-enable autostart with `maestral autostart -Y|-N` to
+  replace old unit files.
+- Fixes a possible race condition when creating the cache directory.
+- Fixes error handling when a file is changed while uploading.
+
+#### Removed:
+
+- Support for config names with spaces. Spaces could cause issues with autostart entries
+  on some platforms.
+- The ability to run the daemon in a separate thread. The daemon must now always be run in
+  its own process.
+
+#### Dependencies:
+
+- Replaced `jeepney` dependency on Linux with `dbus-next`.
+
 ## v1.1.0
 
 This release expands the CLI functionality and improves the handling of file modification
@@ -8,8 +90,8 @@ login items.
 
 #### Added:
 
-- Added `--include-deleted | -d` option to `maestral ls`.
-- Added `--long | -l` option to `maestral ls` to include metadata in listing.
+- Added `--include-deleted` option to `maestral ls`.
+- Added `--long` option to `maestral ls` to include metadata in listing.
 - Added `maestral revs` command to list revisions of a file.
 - Added `maestral restore` command to restore an old revision of a file.
 

README.md

@@ -71,7 +71,7 @@ commands. The most important are:
 - `maestral gui`: Starts the Maestral GUI. Creates a sync daemon if not already running.
 - `maestral start|stop`: Starts or stops the Maestral sync daemon.
 - `maestral pause|resume`: Pauses or resumes syncing.
-- `maestral autostart`: Sets the daemon to start on log in.
+- `maestral autostart -Y|-N`: Sets the daemon to start on log in.
 - `maestral status`: Gets the current status of Maestral.
 - `maestral file-status LOCAL_PATH`: Gets the sync status of an individual file or folder.
 - `maestral excluded add|remove|list`: Command group to manage excluded folders.
@@ -112,7 +112,7 @@ new CLI commands, others require more experience, such as packaging for non-macO
 Look out for issues marked with "good first issue" or "help wanted". Pull requests should be
 made against the develop branch.
 
-Revelant resources are:
+Relevant resources are:
 
 - [Maestral API docs](https://maestral.readthedocs.io)
 - [Dropbox API docs](https://www.dropbox.com/developers/documentation/http/documentation)
@@ -133,7 +133,7 @@ month to offset the cost of an Apple Develper account to sign and notiarize the
 # Acknowledgements
 
 - The config module uses code from the [Spyder IDE](https://github.com/spyder-ide)
-- The MaestralApiClient is based on work from [Orphilia](https://github.com/ksiazkowicz/orphilia-dropbox)
+- The DropboxClient is inspired by work from [Orphilia](https://github.com/ksiazkowicz/orphilia-dropbox)
 - Error reporting is powered by bugsnag:
 
 &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; <a href="https://bugsnag.com"> <img src="https://global-uploads.webflow.com/5c741219fd0819540590e785/5c741219fd0819856890e790_asset%2039.svg" title="Bugsnag text" height="20"></a>

docs/api/sync.rst

@@ -2,7 +2,79 @@
 Sync module
 ===========
 
+This module is the heart of Maestral, it contains the classes for sync functionality.
+
+The :class:`SyncEngine` provides access to the current sync state through its properties
+and provides the methods which are necessary to complete an upload or a download sync
+cycle. This includes methods to wait for and return local and remote changes, to sort
+those changes and discard any excluded items and to apply local changes to the Dropbox
+server and vice versa.
+
+The :class:`SyncMonitor` actually coordinates the sync process with its threads for
+startup, download-sync, upload-sync and periodic maintenance.
+
+Notes on event processing:
+
+Remote events come in three types: DeletedMetadata, FolderMetadata and FileMetadata.
+The Dropbox API does not differentiate between created, moved or modified events.
+Maestral processes remote events as follows:
+
+  1) :meth:`SyncEngine.wait_for_remote_changes` blocks until remote changes are available.
+  2) :meth:`SyncEngine.get_remote_changes` lists all remote changes since the last sync.
+     Events for entries which are excluded by selective sync and hard-coded file names
+     which are always excluded (e.g., '.DS_Store') are filtered out at this stage.
+     Multiple events per file path are combined to one. This is rarely necessary, Dropbox
+     typically already provides only a single event per path but this is not guaranteed
+     and may change. One exception is sharing a folder: This is done by removing the
+     folder from Dropbox and re-mounting it as a shared folder and produces at least one
+     DeletedMetadata and one FolderMetadata event. If querying for changes *during* this
+     process, multiple DeletedMetadata events may be returned. If a file / folder event
+     implies a type changes, e.g., replacing a folder with a file, we explicitly generate
+     the necessary DeletedMetadata here to simplify conflict resolution.
+  3) :meth:`SyncEngine.apply_remote_changes`: Sorts all events hierarchically, with
+     top-level events coming first. Deleted and folder events are processed in order, file
+     events in parallel with up to 6 worker threads. Sync conflicts are detected by
+     comparing the file "rev" with our locally saved rev. We assign folders a rev of
+     ``'folder'`` and deleted / non-existent items a rev of ``None``. If revs are equal,
+     the local item is the same or newer as on Dropbox and no download / deletion occurs.
+     If revs are different, we compare content hashes. Folders are assigned a hash of
+     'folder'. If hashes are equal, no download occurs. If they are different, we check if
+     the local item has been modified since the last download sync. In case of a folder,
+     we take the newest change of any of its children. If the local entry has not been
+     modified since the last sync, it will be replaced. Otherwise, we create a conflicting
+     copy.
+  4) :meth:`SyncEngine.notify_user`: Shows a desktop notification for the remote changes.
+
+Local file events come in eight types: For both files and folders we collect created,
+moved, modified and deleted events. They are processed as follows:
+
+  1) :meth:`SyncEngine.wait_for_local_changes`: Blocks until local changes are registered
+     by :class:`FSEventHandler` and returns those changes. Events ignored by a "mignore"
+     pattern as well as hard-coded file names and changes in our cache path are filtered
+     out at this stage. Further, event are cleaned up to return the minimum number
+     necessary to reproduce the actual changes: Multiple events per path are combined into
+     a single event which reproduces the file change. The only exception is when the entry
+     type changes from file to folder or vice versa: in this case, both deleted and
+     created events are kept. Further, when a whole folder is moved or deleted, we discard
+     the moved or deleted events for its children.
+  2) :meth:`SyncEngine.apply_local_changes`: Sorts local changes hierarchically and
+     applies events in the order of deleted, folders and files. Deletions and creations
+     will be carried out in parallel with up to 6 threads. Conflict resolution and the
+     actual upload will be handled as follows: For created and moved events, we check if
+     the new path has been excluded by the user with selective sync but still exists on
+     Dropbox. If yes, it will be renamed by appending "(selective sync conflict)". On
+     case-sensitive file systems, we check if the new path differs only in casing from an
+     existing path. If yes, it will be renamed by appending "(case conflict)". If a file
+     has been replaced with a folder or vice versa, we check if any un-synced changes will
+     be lost by replacing the remote item and create a conflicting copy if necessary.
+     Dropbox does not handle conflict resolution for us in this case. For created or
+     modified files, check if the local content hash equals the remote content hash. If
+     yes, we don't upload but update our rev number. If no, we upload the changes and
+     specify the rev which we want to replace or delete. If the remote item is newer
+     (different rev), Dropbox will handle conflict resolution for us. We finally confirm
+     the successful upload and check if Dropbox has renamed the item to a conflicting
+     copy. In the latter case, we apply those changes locally.
+
 .. automodule:: sync
    :members:
-   :private-members:
    :show-inheritance:

docs/api/utils/index.rst

@@ -9,6 +9,9 @@ maestral.utils
    utils.autostart <autostart>
    utils.housekeeping <housekeeping>
    utils.notify <notify>
+   utils.notify_base <notify_base>
+   utils.notify_macos <notify_macos>
+   utils.notify_linux <notify_linux>
    utils.path <path>
    utils.serializer <serializer>
    utils.updates <updates>

docs/api/utils/notify_base.rst

@@ -0,0 +1,7 @@
+Desktop notifications base
+==========================
+
+.. automodule:: utils.notify_base
+   :members:
+   :show-inheritance:
+

docs/conf.py

@@ -34,8 +34,9 @@
     'sphinx.ext.ifconfig',
     'sphinx.ext.viewcode',
     'sphinx.ext.githubpages',
-    'm2r',
+    'sphinx_autodoc_typehints',
     'sphinx_click.ext',
+    'm2r'
 ]
 
 templates_path = ['_templates']
@@ -44,8 +45,7 @@
 master_doc = 'index'
 language = None
 pygments_style = None
-
-autodoc_member_order = 'bysource'
+html4_writer = True
 
 # -- Options for HTML output -------------------------------------------------
 
@@ -67,6 +67,8 @@
 
 # -- Extension configuration -------------------------------------------------
 
+autodoc_member_order = 'bysource'
+
 # -- Options for intersphinx extension ---------------------------------------
 
 intersphinx_mapping = {'https://docs.python.org/': None}

docs/requirements.txt

@@ -1,19 +1,5 @@
-atomicwrites
-bugsnag
-click
-dropbox
-fasteners
-importlib_metadata
-keyring
-keyrings.alt
-packaging
-pathspec
-Pyro5
-requests
-sdnotify
-setuptools
-six
-u-msgpack-python
-watchdog
+maestral
 m2r
+sphinx==2.4.4
 sphinx-click
+sphinx-autodoc-typehints