Merge pull request #153 from discoverygarden/feature/pgsql-locking

DGI9-584: Feature/pgsql locking

dgi_migrate.services.yml

@@ -4,3 +4,6 @@ services:
     class: Drupal\dgi_migrate\Routing\RouteSubscriber
     tags:
       - name: event_subscriber
+  plugin.manager.dgi_migrate.locker:
+    class: \Drupal\dgi_migrate\LockerPluginManager
+    parent: default_plugin_manager

scripts/README.md

@@ -106,6 +106,15 @@ NOTE: In terms of establishing the `PROCESSES` quantity, there are environmental
 * Is Crayfish on the same machine? Are derivatives enabled? There can be additional load from Crayfish acquiring files from Drupal, or if on the same machine, from the derivatives proper being run.
 * Is the site being used by others? If so, it is probably a good idea to play nice and to try to avoid saturating the CPUs, perhaps going so far as to `nice` the migration execution. If not, we could target a slight bit of oversaturation, with the expectation that there will be some background I/O overhead on read/write operations that might leave some CPU cycles otherwise unoccupied.
 
+#### Locking plugins
+
+The means of locking is pluggable. The default implementation uses a directory of locks for various purposes in `temporary://`, using [PHP's `\SplFileInfo::flock()`](https://www.php.net/manual/en/splfileobject.flock.php), in [our `flock` plugin](../src/Plugin/dgi_migrate/locker/Flock.php); this `flock` plugin _does_ requires the ability to open many files (on the orders of tens-of-thousands). We also provide [a `pgsql_advisory_locking` plugin](../src/Plugin/dgi_migrate/locker/PgsqlAdvisoryLocking.php) that might be used to perform locking using [a PostgreSQL database's advisory locking capabilities](https://www.postgresql.org/docs/current/functions-admin.html#FUNCTIONS-ADVISORY-LOCKS); however, it is expected that using such might perform somewhat slower than `flock` due to requiring the round-trips to the DB. Ideally, we should look at implementing native semaphore/mutex functionality.
+
+The locking plugin can be:
+- configured by setting the `DGI_MIGRATE_DEFAULT_LOCKER` to the ID of the desired plugin
+  - Built-in are `flock` and `pgsql_advisory_locking`; however, other modules might provide other plugins
+- configured for on the particular migration lookup plugin definition, via the `locker` property.
+
 ### Rollback
 
 If additional parameters/options need to be passed to the `dgi-migrate:rollback`

src/Attribute/Locker.php

@@ -0,0 +1,13 @@
+<?php
+
+namespace Drupal\dgi_migrate\Attribute;
+
+use Drupal\Component\Plugin\Attribute\AttributeBase;
+
+/**
+ * The dgi_migrate "locker" plugin attribute.
+ */
+#[\Attribute(\Attribute::TARGET_CLASS)]
+final class Locker extends AttributeBase {
+
+}

src/LockerPluginManager.php

@@ -0,0 +1,44 @@
+<?php
+
+namespace Drupal\dgi_migrate;
+
+use Drupal\Component\Plugin\FallbackPluginManagerInterface;
+use Drupal\Core\Cache\CacheBackendInterface;
+use Drupal\Core\Extension\ModuleHandlerInterface;
+use Drupal\Core\Plugin\DefaultPluginManager;
+use Drupal\dgi_migrate\Attribute\Locker;
+use Drupal\dgi_migrate\Plugin\dgi_migrate\locker\LockerInterface;
+
+/**
+ * Locker plugin manager service.
+ */
+final class LockerPluginManager extends DefaultPluginManager implements FallbackPluginManagerInterface, LockerPluginManagerInterface {
+
+  /**
+   * Constructor.
+   */
+  public function __construct(
+    \Traversable $namespaces,
+    CacheBackendInterface $cacheBackend,
+    ModuleHandlerInterface $module_handler,
+  ) {
+    parent::__construct(
+      'Plugin/dgi_migrate/locker',
+      $namespaces,
+      $module_handler,
+      LockerInterface::class,
+      Locker::class,
+    );
+
+    $this->alterInfo('dgi_migrate__locker_info');
+    $this->setCacheBackend($cacheBackend, 'dgi_migrate__locker_plugins');
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public function getFallbackPluginId($plugin_id, array $configuration = []) : string {
+    return 'flock';
+  }
+
+}

src/LockerPluginManagerInterface.php

@@ -0,0 +1,12 @@
+<?php
+
+namespace Drupal\dgi_migrate;
+
+use Drupal\Component\Plugin\PluginManagerInterface;
+
+/**
+ * Locker plugin manager interface definition.
+ */
+interface LockerPluginManagerInterface extends PluginManagerInterface {
+
+}

src/Plugin/dgi_migrate/locker/Flock.php

@@ -0,0 +1,113 @@
+<?php
+
+namespace Drupal\dgi_migrate\Plugin\dgi_migrate\locker;
+
+use Drupal\Core\File\FileSystemInterface;
+use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
+use Drupal\Core\Plugin\PluginBase;
+use Drupal\dgi_migrate\Attribute\Locker;
+use Drupal\dgi_migrate\Plugin\migrate\process\LockingMigrationLookup;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+/**
+ * Base implementation using flock.
+ */
+#[Locker('flock')]
+class Flock extends PluginBase implements LockerInterface, ContainerFactoryPluginInterface {
+
+  /**
+   * An array of SplFileObjects, to facilitate locking.
+   *
+   * @var \SplFileObject[]
+   */
+  protected array $lockFiles = [];
+
+  /**
+   * Constructor.
+   */
+  public function __construct(
+    array $configuration,
+    $plugin_id,
+    $plugin_definition,
+    protected FileSystemInterface $fileSystem,
+  ) {
+    parent::__construct($configuration, $plugin_id, $plugin_definition);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) {
+    return new static(
+      $configuration,
+      $plugin_id,
+      $plugin_definition,
+      $container->get('file_system'),
+    );
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public function acquireLock(string $name, int $mode = LOCK_EX, bool &$would_block = FALSE): bool {
+    return $this->getLockFile($name)->flock($mode, $would_block);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public function releaseLock(string $name): bool {
+    return $this->getLockFile($name)->flock(LOCK_UN);
+  }
+
+  /**
+   * Get an \SplFileObject instance to act as the lock.
+   *
+   * @param string $name
+   *   The name of the lock to acquire. Should result in a file being created
+   *   under the temporary:// scheme of the same name, against which `flock`
+   *   commands will be issued.
+   *
+   * @return \SplFileObject
+   *   The \SplFileObject instance against which to lock.
+   */
+  protected function getLockFile(string $name) : \SplFileObject {
+    if (!isset($this->lockFiles[$name])) {
+      $file_uri = "temporary://{$name}";
+      $directory = $this->fileSystem->dirname($file_uri);
+      $basename = $this->fileSystem->basename($file_uri);
+      $this->fileSystem->prepareDirectory($directory, FileSystemInterface::CREATE_DIRECTORY);
+      $file_uri = "{$directory}/{$basename}";
+
+      // XXX: Drupal's LocalStream wrappers presently have a bug in their
+      // ::stream_lock() method which underlies flock()/\SplFileObject::flock(),
+      // where they fail to properly report the lock status when non-blockingly
+      // acquiring locks, so let's side-step the issue by referencing the real
+      // file path directly.
+      //
+      // @see https://www.drupal.org/project/drupal/issues/3493632
+      // @see https://github.com/php/doc-en/issues/4299
+      $file_path = $this->fileSystem->realpath($file_uri);
+
+      touch($file_path);
+      $this->lockFiles[$name] = new \SplFileObject($file_path, 'a+');
+    }
+
+    return $this->lockFiles[$name];
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public function acquireControl(): bool {
+    return $this->acquireLock(LockingMigrationLookup::CONTROL_LOCK);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public function releaseControl(): bool {
+    return $this->releaseLock(LockingMigrationLookup::CONTROL_LOCK);
+  }
+
+}

src/Plugin/dgi_migrate/locker/LockerInterface.php

@@ -0,0 +1,58 @@
+<?php
+
+namespace Drupal\dgi_migrate\Plugin\dgi_migrate\locker;
+
+/**
+ * Interface for locker plugins.
+ *
+ * Intended to be very similar to flock(), without being strictly bound to
+ * files.
+ */
+interface LockerInterface {
+
+  /**
+   * Acquire lock of given name.
+   *
+   * @param string $name
+   *   The name/ID to lock.
+   * @param int $mode
+   *   The mode with which to lock, as a bit-field, expecting the use of the
+   *   LOCK_EX, LOCK_SH and LOCK_NB constants.
+   * @param bool $would_block
+   *   If LOCK_NB was in $mode, flag if we failed to acquire the lock due to it
+   *   being held by another process.
+   *
+   * @return bool
+   *   TRUE if we acquired the lock; otherwise, FALSE.
+   */
+  public function acquireLock(string $name, int $mode = LOCK_EX, bool &$would_block = FALSE) : bool;
+
+  /**
+   * Release lock of the given name.
+   *
+   * @param string $name
+   *   The name/ID of the lock to release.
+   *
+   * @return bool
+   *   TRUE if we released the lock; otherwise, FALSE (if we did not hold the
+   *   given lock?).
+   */
+  public function releaseLock(string $name) : bool;
+
+  /**
+   * Acquire control lock.
+   *
+   * @return bool
+   *   TRUE if it was successfully acquired; otherwise, FALSE.
+   */
+  public function acquireControl() : bool;
+
+  /**
+   * Release control lock.
+   *
+   * @return bool
+   *   TRUE if it was successfully released; otherwise, FALSE.
+   */
+  public function releaseControl() : bool;
+
+}