Merge pull request #3 from ingenerator/sql-dump-importer

Add a SqlDumpImporter utility class
ingenerator · Oct 24, 2022 · 1e5522b · 1e5522b
2 parents 682f8fd + fb68558
commit 1e5522b
Show file tree

Hide file tree

Showing 4 changed files with 361 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,7 +1,8 @@
 # Version History
 
-** UNRELEASED **
+** 1.0.0 ** (2022-10-24)
 
+* Add a `SqlDumpImporter` utility class to have a migration load in statements from a standard SQL dump file.
 * Remove support for 'wrapping' database adapters, and the `TimedOutputAdapter`. The timed adapter was the only
   remaining implementation of this proxy pattern, and was only actually printing statistics for `insert()` operations.
   This was not that useful, and the overhead (e.g. on identifying dead code / improving type safety / adding to

diff --git a/composer.json b/composer.json
@@ -52,7 +52,8 @@
         "symfony/yaml": "^4.0"
     },
     "require-dev": {
-        "phpunit/phpunit": "^9.5.5"
+        "phpunit/phpunit": "^9.5.25",
+        "mikey179/vfsstream": "^1.6.11"
     },
     "conflict": {
         "robmorgan/phinx": "*"

diff --git a/src/Phinx/Util/SqlDumpImporter.php b/src/Phinx/Util/SqlDumpImporter.php
@@ -0,0 +1,93 @@
+<?php
+
+namespace Phinx\Util;
+
+use Phinx\Db\Adapter\AdapterInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class SqlDumpImporter
+{
+
+    public function __construct(
+        private AdapterInterface $adapter,
+        private OutputInterface $output
+    ) {
+    }
+
+    public function import(string $file, int $max_statement_length = 2000000): void
+    {
+        $index = 0;
+        foreach ($this->iterateSqlStatements($file, $max_statement_length) as $index => $statement) {
+            try {
+                $this->adapter->execute($statement);
+            } catch (\Exception $e) {
+                $this->output->writeln(sprintf('Failed importing %s - failing statement #%d:', $file, $index));
+                $this->output->writeln($statement);
+                throw $e;
+            }
+
+            if ($index % 20 === 0) {
+                $this->output->write('.');
+            }
+        }
+
+        if ($index === 0) {
+            throw new \RuntimeException(sprintf("File %s is empty", $file));
+        }
+
+        // Close off the `.` progress line
+        $this->output->write("\n");
+        $this->output->writeln(sprintf('%s statements executed from %s', $index, $file));
+    }
+
+    private function iterateSqlStatements(string $file, int $max_statement_length)
+    {
+        $fp    = $this->openFileForReading($file);
+        $index = 1;
+        try {
+            while ( ! feof($fp)) {
+                $statement = stream_get_line($fp, $max_statement_length, ";\n");
+
+                // stream_get_line does not return the delimiter so can't tell if it actually hit the max buffer size.
+                // However it's fairly unlikely the statement is identical to the buffer size unless it was truncated.
+                // If you hit this, either reduce the length of individual statements in the SQL file, or increase the
+                // buffer size to accommodate them.
+                if (strlen($statement) === $max_statement_length) {
+                    throw new \RuntimeException(
+                        sprintf(
+                            "Statement %s was exactly the buffer size (%s bytes) - most likely it was truncated?\nGot:\n%s",
+                            $index,
+                            $max_statement_length,
+                            $statement
+                        )
+                    );
+                }
+
+                if ($statement) {
+                    yield $index => $statement;
+                    $index++;
+                }
+            }
+        } finally {
+            fclose($fp);
+        }
+    }
+
+    /**
+     * @param string $file
+     *
+     * @return resource
+     */
+    private function openFileForReading(string $file)
+    {
+        try {
+            \set_error_handler(fn(int $type, string $msg) => throw new \RuntimeException("fopen failed: $msg ($type)"));
+
+            return fopen($file, 'r');
+        } catch (\Throwable $e) {
+            throw new \RuntimeException(sprintf('Could not open import file "%s": %s', $file, $e->getMessage()), 0, $e);
+        } finally {
+            \restore_error_handler();
+        }
+    }
+}
diff --git a/tests/Phinx/Util/SqlDumpImporterTest.php b/tests/Phinx/Util/SqlDumpImporterTest.php
@@ -0,0 +1,264 @@
+<?php
+
+namespace Test\Phinx\Util;
+
+use org\bovigo\vfs\vfsStream;
+use Phinx\Db\Adapter\AdapterInterface;
+use Phinx\Db\Adapter\MysqlAdapter;
+use Phinx\Util\SqlDumpImporter;
+use PHPUnit\Framework\TestCase;
+use Symfony\Component\Console\Output\BufferedOutput;
+use Symfony\Component\Console\Output\NullOutput;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class SqlDumpImporterTest extends TestCase
+{
+
+    private AdapterInterface $adapter;
+
+    private OutputInterface $output;
+
+    public function providerStatements()
+    {
+        return [
+            'single statement, no terminator'                                => [
+                <<<SQL
+                CREATE TABLE `foo` (
+                    `whatever` INT NOT NULL
+                ) ENGINE=INNODB
+                SQL,
+                [
+                    <<<SQL
+                    CREATE TABLE `foo` (
+                        `whatever` INT NOT NULL
+                    ) ENGINE=INNODB
+                    SQL,
+                ],
+            ],
+            'multiple statements more like a dump file'                      => [
+                <<<SQL
+                /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+                /*!40103 SET TIME_ZONE='+00:00' */;
+                /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+                /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+                /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+                /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+                /*!40101 SET @saved_cs_client     = @@character_set_client */;
+                /*!40101 SET character_set_client = utf8 */;
+                CREATE TABLE `customers` (
+                  `id` int(11) NOT NULL,
+                  PRIMARY KEY (`id`)
+                ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
+                /*!40101 SET character_set_client = @saved_cs_client */;
+                SQL,
+                [
+                    '/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */',
+                    '/*!40103 SET TIME_ZONE=\'+00:00\' */',
+                    '/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */',
+                    '/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */',
+                    '/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE=\'NO_AUTO_VALUE_ON_ZERO\' */',
+                    '/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */',
+                    '/*!40101 SET @saved_cs_client     = @@character_set_client */',
+                    '/*!40101 SET character_set_client = utf8 */',
+                    <<<SQL
+                    CREATE TABLE `customers` (
+                      `id` int(11) NOT NULL,
+                      PRIMARY KEY (`id`)
+                    ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+                    SQL,
+                    '/*!40101 SET character_set_client = @saved_cs_client */;',
+                ],
+            ],
+            'comments and empty lines are included with following statement' => [
+                <<<SQL
+                /*!40101 SET character_set_client = @saved_cs_client */;
+
+                --
+                -- Table structure for table `customers`
+                --
+
+                /*!40101 SET @saved_cs_client     = @@character_set_client */;
+                /*!40101 SET character_set_client = utf8 */;
+                SQL,
+                [
+                    '/*!40101 SET character_set_client = @saved_cs_client */',
+                    <<<SQL
+
+                    --
+                    -- Table structure for table `customers`
+                    --
+
+                    /*!40101 SET @saved_cs_client     = @@character_set_client */
+                    SQL,
+                    '/*!40101 SET character_set_client = utf8 */;',
+                ],
+            ],
+        ];
+    }
+
+    /**
+     * @dataProvider providerStatements
+     */
+    public function testItExecutesEachStatementIndividually($content, $expectStatements)
+    {
+        $path          = $this->givenSqlFileWithContent($content);
+        $this->adapter = new class extends MysqlAdapter {
+            public array $executed = [];
+
+            public function __construct() { }
+
+            public function execute(string $sql): false|int
+            {
+                $this->executed[] = $sql;
+
+                return 0;
+            }
+        };
+
+        $this->newSubject()->import($path);
+
+        $this->assertSame($expectStatements, $this->adapter->executed);
+    }
+
+    public function testItIsInitialisable()
+    {
+        $this->assertInstanceOf(SqlDumpImporter::class, $this->newSubject());
+    }
+
+    private function newSubject(): SqlDumpImporter
+    {
+        return new SqlDumpImporter($this->adapter, $this->output);
+    }
+
+    public function testItLogsFailingStatementAndRethrows()
+    {
+        $path          = $this->givenSqlFileWithContent(
+            <<<SQL
+            /*!40101 SET @saved_cs_client     = @@character_set_client */;
+            /*!40101 SET character_set_client = utf8 */;
+
+            CREATE TABLE `foo` (
+                `column` BADTYPE JUNK DEFINITION
+            ) ENGINE=nothing;
+
+            /*!40101 SET character_set_client = utf8 */;
+            SQL
+        );
+        $mockException = new \RuntimeException('SQLSTATE Whatever');
+        $this->adapter = new class($mockException) extends MysqlAdapter {
+            public function __construct(private \Exception $exception) { }
+
+            public function execute(string $sql): false|int
+            {
+                if (\str_contains($sql, 'BADTYPE')) {
+                    throw $this->exception;
+                }
+
+                return 0;
+            }
+        };
+        $this->output  = new BufferedOutput(decorated: FALSE);
+
+        try {
+            $this->newSubject()->import($path);
+            $this->fail('Expected exception, none got');
+        } catch (\RuntimeException$e) {
+            $this->assertSame($e, $mockException, 'Expected exception to be rethrown');
+        }
+
+        $this->assertSame(
+            <<<LOG
+            Failed importing $path - failing statement #3:
+
+            CREATE TABLE `foo` (
+                `column` BADTYPE JUNK DEFINITION
+            ) ENGINE=nothing
+
+            LOG,
+            $this->output->fetch()
+        );
+    }
+
+    public function testItThrowsIfFileIsEmpty()
+    {
+        $path = $this->givenSqlFileWithContent('');
+
+        $this->expectException(\RuntimeException::class);
+        $this->expectExceptionMessage('is empty');
+
+        $this->newSubject()->import($path);
+    }
+
+    private function givenSqlFileWithContent(string $fileContent): string
+    {
+        $vfs  = vfsStream::setup('import', NULL, [
+            'some-sql.sql' => $fileContent,
+        ]);
+        $path = $vfs->getChild('some-sql.sql')->url();
+
+        return $path;
+    }
+
+    public function testItThrowsIfFileNotExists()
+    {
+        $vfs               = vfsStream::setup('import');
+        $missing_file_path = $vfs->url().'/some-file.sql';
+        $subject           = $this->newSubject();
+
+        $this->expectException(\RuntimeException::class);
+        $this->expectExceptionMessage('some-file.sql');
+        $subject->import($missing_file_path);
+    }
+
+    protected function setUp(): void
+    {
+        parent::setUp();
+        $this->adapter = $this->getMockBuilder(AdapterInterface::class)->getMock();
+        $this->output  = new NullOutput;
+    }
+
+    public function testItThrowsIfStatementLineTooLong()
+    {
+        $path = $this->givenSqlFileWithContent(
+            <<<SQL
+            /*!40101 SET @saved_cs_client     = @@character_set_client */;
+            /*!40101 SET character_set_client = utf8 */;
+
+            INSERT INTO `whatever`
+            (`foo`, `bar`)
+            VALUES
+            ('something long'),
+            ('something else long'),
+            ('something even longer');
+
+            SELECT 'some short thing' FROM `whatever`;
+
+            SQL
+        );
+
+        $this->expectException(\RuntimeException::class);
+        $this->expectExceptionMessage(
+            'Statement 3 was exactly the buffer size (110 bytes) - most likely it was truncated'
+        );
+
+        $this->newSubject()->import($path, max_statement_length: 110);
+    }
+
+    public function testItWritesRegularProgressAndSummaryAtEnd()
+    {
+        $queries      = \str_repeat("/* SELECT anything */;\n", 45);
+        $path         = $this->givenSqlFileWithContent($queries);
+        $this->output = new BufferedOutput;
+        $this->newSubject()->import($path);
+
+        $this->assertSame(
+            <<<LOG
+            ..
+            45 statements executed from $path
+
+            LOG,
+            $this->output->fetch()
+        );
+    }
+
+}