From 7f24f2471a5189d1d04b43da1ba51e21912ee6f5 Mon Sep 17 00:00:00 2001 From: Greg Anderson Date: Thu, 7 Jul 2022 10:45:21 -0500 Subject: [PATCH] Jabxjab archive:dump and archive:restore commands (#5148) * Add ArchiveCommands.php * Add SQL dump operation * Add "description", "tags", "destination" and "overwrite" options to annotation * Refactor getDrupalFilesDir(); Create database.tar sub-archive; Create files.tar sub-archive; Create master archive.tar.gz * Refactor creating database and file archives * Add createMasterArchive() * Add createManifestFile() * Rename "db_dump" option into "db" * Update default generator value * Add "--destination" and "--overwrite" options * Update info messages * Log info when creating manifest file * Remove "master" archive mentions in logs * Refactor archive structure ("database" and "files" are now directories) * Add "code" component * Add getExcludesByPaths() * Refactor code excludes * Exclude css, js, php and styles from file component * Exclude from code archive everything from web/ except modules, themes, profiles * Refactor Drupal contrib projects excludes * Exclude sites/*/settings.local.php from code archive * Add validateSensitiveData() * Remove todo * Fix method name * Fix getCodeComponentPath() * Rename getCodeComponentPath() into getProjectPath() * Add site validation * Update command description * Fix validateSite() * Add ArchiveTest * Fix code style issues * Add support for non-Composer managed, non-web docroot sites * Fix command for non-web docroot sites * Exclude settings.*.php * Fix testArchiveDumpCommand() test * Add cleanUp() shutdown function Use Symfony Filesystem component * Copy code archive component to archive dir * Use constants for archive component names * Refactor excludes * Simplify cleanUp() * Rename constants * Rename getExcludesByPaths() into getRegexpsForPaths() * Update phpdocs * Rename var * Use Path::join() * Use array_map in getRegexpsForPaths() * Rename getIterator() * Fix missing manifest file * Return absolute path to the archive * Delete temp archive.tar.gz if exists * Add scenarios to testArchiveDumpCommand() * Fix code style issues * Fix testArchiveDumpCommand() * Rename ArchiveDumpCommands and refactor getDrupalFilesComponentPath() * Remove setSimulated() * Set php memory_limit to 256M in appveyor.yml * Fix file iterator local file path handling * Remove unused constants * Create temp dir on command execution only * Add ArchiveRestoreCommands backbone * Add confirmation prompts * Implement extractArchive() * Add check for file existence * Add check for filename extension * Add todo * Import code file for a local target only * Import code files for a remote targets * Add 'docroot' property to the manifest file * Fix $aliasConfigContext * Fix nested php includes in validateSensitiveData() * Bootstrap Drupal core to get a path to Drupal public files * Disable type3/phar-stream-wrapper which loads with Drupal core and prevents creating *.tar archives * Fix getDrupalFilesComponentPath() * Remove Site Alias manager * Refactor validateSensitiveData() to include site/*/settings.php file without include/require directives * Fix scenario when backup dir is inside code dir (and on Windows OS) * Fix scenario when backup dir is inside code dir (and on Windows OS) using \Webmozart\PathUtil\Path util * Fix getCodeComponentPath() * Improve verbosity in validateSensitiveData() * Add dt() to Exception messages * Refactor Drupal-related excludes - exclude paths defined in composer.json in "extra"/"installer-paths" section * Detect Drupal files exclude for code component * Use 'composer info --path --format=json' to exclude packages installation paths * Add "composer" as alias to "php composer.phar" in appveyor.yml * Rename $installedPackagesBaseDirs into $installedPackagesRelativePaths * Use native Symfony\Process for "composer info" command execution * Add '%composer-root' to core:status command * Add 'composer-root' to @field-labels/@default-fields of core:status command * Detect composer root for a remote site * Add rsyncFiles() * Add importFiles() * Add getSiteAlias(), getSiteStatus() * Fix $evaluatedPath instantiation for a remote site * Detect composer root for a remote site * Add rsyncFiles() * Add importFiles() * Add getSiteAlias(), getSiteStatus() * Fix $evaluatedPath instantiation for a remote site * Add blueprint of importDatabase() * Update importDatabase() * Fix todos * Fix code style issues * Add testArchiveRestoreCommand() to ArchiveTest * Add test scenario for "--overwrite" option * Add test scenario for "--code", "--code_path", "--db", "--db_path", "--files", "--files_path" options * PROPOSAL: Replace direct use of register_shutdown_function with Robo _tmpDir() facility. * Proposal 2: Provide our own tmpDir service in FsUtils * Code style * Simplify tests: do one less archive:dump, and instead use file_put_contents to test with and without --overwrite. * Update download url of cacert to new location * Try curl instead of DownloadFile * Skip tests of unsupported configurations * Remove eval() from validateSensitiveData() * Remove support for a remote site for archive:restore command * Add an option to restore code into current dir if empty * Remove ssh-related option * Add --destination_path option * Rename archive:restore options * Do not check $destination dir (creates automatically by rsync) * Throw exception on Drupal bootstrap failure when getting Drupal files path * Add archive:restore options: db-name, db-port, db-host, db-user, db-password, db-prefix, db-driver * Replace underscores with dashes in option names * Update tests for archive:restore * Remove $autodetectDestination * Fix indentations * Disable SecurityUpdatesTest.php temporarily * Require database connection option(s) when --destination-path options is provided * Add support for relative paths in --destination-path option * Add test scenarios for testArchiveRestoreCommand() * Set sut db password to empty string * Use self::getDbUrl() to construct --db-url * Add tearDown() to delete a test file * Update testing scenario to use invalid db host instead of password * Add --db-driver option in tests * Add testing scenarios for database component involving --destination-path option * Revert "Disable SecurityUpdatesTest.php temporarily" This reverts commit f7e66e9cbcdc02f67dd45a69788c11e8962c56f8. * Allow archive:restore to extract archive and files to cwd. Improve FsUtils::tmpDir, and use it in archive:restore. Add default options to sut to avoid including Drush sources in archive:dumps of the sut, for more convenient ad-hoc testing. * Code style * WIP: Avoid overwriting sut in archive restore tests * Fix var name typo * Remove unused namespace, @throws tag and $root var * Remove unused namespaces and ARCHIVE_SUBDIR_NAME const * Refactor extractArchive() into getExtractDir() * Create database if not exists database import * Remove duplicated testing scenarios * Update archive:restore tests * Remove tearDown() * Use setDbSpec() before calling query() in ArchiveRestoreCommands::importDatabase() * Fix drop/create db in importDatabase() * Remove escaping "\" in --exclude-code-paths * Add composer.json and composer.lock to a SUT's archive * Run 'composer install' after archive:restore in testArchiveRestoreCommand() * Manage SUT's settings.php file * Refactor settings.php file setup procedures * Remove unused arg * Delete destination directory in case when code component is involved and add a confirmation dialog * Add confirmation dialog for Drupal files restore if destination path already exists and --overwrite is not set * Fix tests * Show prompt on file import if Drupal files dir exists and one of --code or --overwrite option is not set * Refactor setupSettingsPhp() * Refactor backupSettingsPhp() * Add installComposerDependencies() * Add assertRestoredSiteStatus() * Remove "composer install" code in favor of the method * Remove excessive assertion * Add "composer install" to testArchiveRestoreCommand() * Change $filesRelativePath to a real value * Assert the restored site is OK * Add more assertions that the restored site is OK * Update dependencies to fix testNoInsecureProductionPhpPackage failure. * Remove "composer install" from tests * Rename "destination path" property/method name * Add --code-no-composer-install and --code-composer-install-timeout options * Add "composer install" to importCode() * Remove setupSettingsPhp() and backupSettingsPhp() from tests * Add --site-subdir and --setup-database-connection options * Add getDrupalRootPath() * Update $options default value * Add setupLocalSettingsPhp() * Update testArchiveRestoreCommand() * Fix code style issue * Add saveSettingsLocalPhp() * Add database connection settings to settings.local.php file * Update dependencies to fix testNoInsecureProductionPhpPackage failure. * Fix null command references (in service commands list) during Drupal bootstrap phase (which causes "PHP Fatal error: Cannot redeclare drupal_error_levels() (previously declared in..." in test_81_drupal10_highest CI pipeline) * Jabxjab archive dump command updates (#5165) * field-delete: Fix field being deleted from all bundles instead of only the requested bundle (#5158) * Updates input default options and provides destination validation. * Removes automatic composer install and adds user feedback. * Resolves phpcs feedback. * Adds composer install to test to allow Drupal installation. * Resolves code sniff. * Adds needed class. * Runs composer update to resolve security warnings. * Updates guzzle version. Co-authored-by: Dieter Holvoet Co-authored-by: Ryan Wagner * Validate that the Phar extension is available * Remove class phpdoc descriptions * Remove unused "use" statement * Remove --code-no-composer-install and --code-composer-install-timeout options * Remove @var phpdocs on explicitly declared class properties * Add @usage examples for archive:restore * Add @usage examples for archive:dump * Validate "rsync" exists for archive:restore command Co-authored-by: Sergei Churilo Co-authored-by: Sergei Churilo Co-authored-by: J Ryan Wagner Co-authored-by: Dieter Holvoet Co-authored-by: Ryan Wagner --- appveyor.yml | 7 +- composer.json | 2 +- composer.lock | 2 +- src/Boot/DrupalBoot8.php | 12 +- src/Command/ServiceCommandlist.php | 2 +- src/Commands/core/ArchiveDumpCommands.php | 605 ++++++++++++++++ src/Commands/core/ArchiveRestoreCommands.php | 716 +++++++++++++++++++ src/Utils/FsUtils.php | 62 +- tests/functional/ArchiveTest.php | 366 ++++++++++ tests/unish/Utils/FSUtils.php | 4 + 10 files changed, 1765 insertions(+), 13 deletions(-) create mode 100644 src/Commands/core/ArchiveDumpCommands.php create mode 100644 src/Commands/core/ArchiveRestoreCommands.php create mode 100644 tests/functional/ArchiveTest.php diff --git a/appveyor.yml b/appveyor.yml index d2aecb76d1..e1315945d4 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -30,7 +30,8 @@ install: - copy php.ini-production php.ini # https://github.com/php-coveralls/php-coveralls/pull/223/files - - appveyor DownloadFile http://curl.haxx.se/ca/cacert.pem -FileName C:\cacert.pem + # - appveyor DownloadFile https://curl.se/ca/cacert.pem -FileName C:\cacert.pem + - curl -fsS -o C:\cacert.pem https://curl.se/ca/cacert.pem - echo curl.cainfo=C:\cacert.pem >> php.ini - echo extension_dir=ext >> php.ini @@ -48,12 +49,14 @@ install: - echo extension=php_pgsql.dll >> php.ini - echo extension=php_gd2.dll >> php.ini - echo extension=php_fileinfo.dll >> php.ini + - echo memory_limit=256M >> php.ini - SET PATH=C:\tools\php74;%PATH% #Install Composer - cd %APPVEYOR_BUILD_FOLDER% #- appveyor DownloadFile https://getcomposer.org/composer.phar - php -r "readfile('http://getcomposer.org/installer');" | php - #Install dependencies via Composer + - echo @php %cd%\composer.phar %%* > composer.bat + # Install dependencies via Composer. - php composer.phar install --prefer-dist -n - SET PATH=%APPVEYOR_BUILD_FOLDER%;%APPVEYOR_BUILD_FOLDER%/vendor/bin;%PATH% # Uncomment this and on_finish line below to enable RDP into build machine https://www.appveyor.com/docs/how-to/rdp-to-build-worker/ diff --git a/composer.json b/composer.json index 36a258e450..a60c675c8d 100644 --- a/composer.json +++ b/composer.json @@ -41,7 +41,7 @@ "consolidation/site-alias": "^3.1.3", "consolidation/site-process": "^4.1.3 || ^5", "enlightn/security-checker": "^1", - "guzzlehttp/guzzle": "^6.3 || ^7.0", + "guzzlehttp/guzzle": "^6.5 || ^7.0", "league/container": "^3.4 || ^4", "psy/psysh": "~0.11", "symfony/event-dispatcher": "^4.0 || ^5.0 || ^6.0", diff --git a/composer.lock b/composer.lock index 586031bf0b..c11783440d 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "eb7c538821997a9476b29e01705ce17f", + "content-hash": "f76084172fce10bbe10655f5ada923d2", "packages": [ { "name": "chi-teck/drupal-code-generator", diff --git a/src/Boot/DrupalBoot8.php b/src/Boot/DrupalBoot8.php index b8465bb5be..85b8bcb552 100644 --- a/src/Boot/DrupalBoot8.php +++ b/src/Boot/DrupalBoot8.php @@ -263,17 +263,17 @@ public function addDrupalModuleDrushCommands($manager): void // Set the command info alterers. if ($container->has(DrushServiceModifier::DRUSH_COMMAND_INFO_ALTERER_SERVICES)) { - $serviceCommandInfoAltererlist = $container->get(DrushServiceModifier::DRUSH_COMMAND_INFO_ALTERER_SERVICES); + $serviceCommandInfoAltererList = $container->get(DrushServiceModifier::DRUSH_COMMAND_INFO_ALTERER_SERVICES); $commandFactory = Drush::commandFactory(); - foreach ($serviceCommandInfoAltererlist->getCommandList() as $altererHandler) { + foreach ($serviceCommandInfoAltererList->getCommandList() as $altererHandler) { $commandFactory->addCommandInfoAlterer($altererHandler); $this->logger->debug(dt('Commands are potentially altered in !class.', ['!class' => get_class($altererHandler)])); } } - $serviceCommandlist = $container->get(DrushServiceModifier::DRUSH_CONSOLE_SERVICES); if ($container->has(DrushServiceModifier::DRUSH_CONSOLE_SERVICES)) { - foreach ($serviceCommandlist->getCommandList() as $command) { + $serviceCommandList = $container->get(DrushServiceModifier::DRUSH_CONSOLE_SERVICES); + foreach ($serviceCommandList->getCommandList() as $command) { $manager->inflect($command); $this->logger->debug(dt('Add a command: !name', ['!name' => $command->getName()])); $application->add($command); @@ -281,8 +281,8 @@ public function addDrupalModuleDrushCommands($manager): void } // Do the same thing with the annotation commands. if ($container->has(DrushServiceModifier::DRUSH_COMMAND_SERVICES)) { - $serviceCommandlist = $container->get(DrushServiceModifier::DRUSH_COMMAND_SERVICES); - foreach ($serviceCommandlist->getCommandList() as $commandHandler) { + $serviceCommandList = $container->get(DrushServiceModifier::DRUSH_COMMAND_SERVICES); + foreach ($serviceCommandList->getCommandList() as $commandHandler) { $manager->inflect($commandHandler); $this->logger->debug(dt('Add a commandfile class: !name', ['!name' => get_class($commandHandler)])); $runner->registerCommandClass($application, $commandHandler); diff --git a/src/Command/ServiceCommandlist.php b/src/Command/ServiceCommandlist.php index e1815e9fde..9c9fb65685 100644 --- a/src/Command/ServiceCommandlist.php +++ b/src/Command/ServiceCommandlist.php @@ -17,6 +17,6 @@ public function addCommandReference($command): void public function getCommandList(): array { - return $this->commandList; + return array_filter($this->commandList); } } diff --git a/src/Commands/core/ArchiveDumpCommands.php b/src/Commands/core/ArchiveDumpCommands.php new file mode 100644 index 0000000000..bd36f31b14 --- /dev/null +++ b/src/Commands/core/ArchiveDumpCommands.php @@ -0,0 +1,605 @@ + false, + 'files' => false, + 'db' => false, + 'destination' => InputOption::VALUE_REQUIRED, + 'overwrite' => false, + 'description' => InputOption::VALUE_REQUIRED, + 'tags' => InputOption::VALUE_REQUIRED, + 'generator' => InputOption::VALUE_REQUIRED, + 'generatorversion' => InputOption::VALUE_REQUIRED, + 'exclude-code-paths' => InputOption::VALUE_REQUIRED, + ]): string + { + $this->prepareArchiveDir(); + + if (!$options['code'] && !$options['files'] && !$options['db']) { + $options['code'] = $options['files'] = $options['db'] = true; + } + + $components = []; + + if ($options['code']) { + $components[] = [ + 'name' => self::COMPONENT_CODE, + 'path' => $this->getCodeComponentPath($options), + ]; + } + + if ($options['files']) { + $components[] = [ + 'name' => self::COMPONENT_FILES, + 'path' => $this->getDrupalFilesComponentPath(), + ]; + } + + if ($options['db']) { + $components[] = [ + 'name' => self::COMPONENT_DATABASE, + 'path' => $this->getDatabaseComponentPath($options), + ]; + } + + return $this->createArchiveFile($components, $options); + } + + /** + * Creates a temporary directory for the archive. + * + * @throws \Exception + */ + protected function prepareArchiveDir(): void + { + $this->filesystem = new Filesystem(); + $this->archiveDir = FsUtils::tmpDir(self::ARCHIVES_DIR_NAME); + } + + /** + * Creates the archive file and returns the absolute path. + * + * @param array $archiveComponents + * The list of components (files) to include into the archive file. + * @param array $options + * The command options. + * + * @return string + * The full path to archive file. + * + * @throws \Exception + */ + private function createArchiveFile(array $archiveComponents, array $options): string + { + if (!$archiveComponents) { + throw new Exception(dt('Nothing to archive')); + } + + $this->logger()->info(dt('Creating archive...')); + $archivePath = Path::join(dirname($this->archiveDir), self::ARCHIVE_FILE_NAME); + + stream_wrapper_restore('phar'); + $archive = new PharData($archivePath); + + $this->createManifestFile($options); + $archive->buildFromDirectory($this->archiveDir); + + $this->logger()->info(dt('Compressing archive...')); + $this->filesystem->remove($archivePath . '.gz'); + $archive->compress(Phar::GZ); + + unset($archive); + Phar::unlinkArchive($archivePath); + $archivePath .= '.gz'; + + if (!$options['destination']) { + return $archivePath; + } + + $options['destination'] = $this->destinationCleanup($options['destination']); + + if ($this->filesystem->exists($options['destination'])) { + if (!$options['overwrite']) { + throw new Exception( + dt('The destination file already exists. Use "--overwrite" option for overwriting an existing file.') + ); + } + + $this->filesystem->remove($options['destination']); + } + + $this->logger()->info( + dt( + 'Moving archive file from !from to !to', + ['!from' => $archivePath, '!to' => $options['destination']] + ) + ); + $this->filesystem->rename($archivePath, $options['destination']); + + return $options['destination']; + } + + /** + * Creates the MANIFEST file. + * + * @param array $options + * The command options. + * + * @throws \Exception + */ + private function createManifestFile(array $options): void + { + $this->logger()->info(dt('Creating !manifest file...', ['!manifest' => self::MANIFEST_FILE_NAME])); + $manifest = [ + 'datestamp' => time(), + 'formatversion' => self::MANIFEST_FORMAT_VERSION, + 'components' => [ + self::COMPONENT_CODE => $options['code'], + self::COMPONENT_FILES => $options['files'], + self::COMPONENT_DATABASE => $options['db'], + ], + 'description' => $options['description'] ?? null, + 'tags' => $options['tags'] ?? null, + 'generator' => $options['generator'] ?? 'Drush archive:dump', + 'generatorversion' => $options['generatorversion'] ?? Drush::getVersion(), + ]; + $manifestFilePath = Path::join($this->archiveDir, self::MANIFEST_FILE_NAME); + file_put_contents( + $manifestFilePath, + Yaml::dump($manifest) + ); + } + + /** + * Returns TRUE if the site is a "web" docroot site. + * + * @return bool + * + * @throws \Exception + */ + private function isWebRootSite(): bool + { + return $this->getComposerRoot() !== $this->getRoot(); + } + + /** + * Returns site's docroot name. + * + * @return string + * + * @throws \Exception + */ + private function getComposerRoot(): string + { + $bootstrapManager = Drush::bootstrapManager(); + $composerRoot = $bootstrapManager->getComposerRoot(); + if (!$composerRoot) { + throw new Exception(dt('Path to Composer root is empty.')); + } + + return $composerRoot; + } + + /** + * Returns site's docroot path. + * + * @return string + * + * @throws \Exception + */ + private function getRoot(): string + { + $bootstrapManager = Drush::bootstrapManager(); + $root = $bootstrapManager->getRoot(); + if (!$root) { + throw new Exception(dt('Path to Drupal docroot is empty.')); + } + + return $root; + } + + /** + * Creates "code" archive component and returns the absolute path. + * + * @param array $options + * The command options. + * + * @return string + * The full path to the code archive component directory. + * + * @throws \Exception + */ + private function getCodeComponentPath(array $options): string + { + $codePath = $this->getComposerRoot(); + $codeArchiveComponentPath = Path::join($this->archiveDir, self::COMPONENT_CODE); + + $this->logger()->info( + dt( + 'Copying code files from !from_path to !to_path...', + ['!from_path' => $codePath, '!to_path' => $codeArchiveComponentPath] + ) + ); + + $excludes = $options['exclude-code-paths'] + ? $this->getRegexpsForPaths(explode(',', $options['exclude-code-paths'])) + : []; + + $excludeDirs = [ + '.git', + 'vendor', + ]; + + $process = Process::fromShellCommandline(sprintf('composer info --path --format=json --working-dir=%s', $this->getComposerRoot())); + $process->mustRun(); + $composerInfoRaw = $process->getOutput(); + $installedPackages = json_decode($composerInfoRaw, true)['installed'] ?? []; + $installedPackagesPaths = array_column($installedPackages, 'path'); + $installedPackagesRelativePaths = array_map( + fn($path) => ltrim(str_replace([$this->getComposerRoot()], '', $path), '/'), + $installedPackagesPaths + ); + $installedPackagesRelativePaths = array_unique( + array_filter( + $installedPackagesRelativePaths, + fn($path) => '' !== $path && 0 !== strpos($path, 'vendor') + ) + ); + $excludeDirs = array_merge($excludeDirs, $installedPackagesRelativePaths); + + if (Path::isBasePath($this->getComposerRoot(), $this->archiveDir)) { + $excludeDirs[] = Path::makeRelative($this->archiveDir, $this->getComposerRoot()); + } + + $excludes = array_merge( + $excludes, + $this->getRegexpsForPaths( + $excludeDirs + ), + $this->getDrupalExcludes() + ); + + $this->filesystem->mirror( + $codePath, + $codeArchiveComponentPath, + $this->getFileIterator($codePath, $excludes) + ); + + return $codeArchiveComponentPath; + } + + /** + * Creates "Drupal files" archive component and returns the absolute path. + * + * @return string + * The full path to the Drupal files archive component directory. + * + * @throws \Exception + */ + private function getDrupalFilesComponentPath(): string + { + $drupalFilesPath = $this->getDrupalFilesDir(); + $drupalFilesArchiveComponentPath = Path::join($this->archiveDir, self::COMPONENT_FILES); + $this->logger()->info( + dt( + 'Copying Drupal files from !from_path to !to_path...', + ['!from_path' => $drupalFilesPath, '!to_path' => $drupalFilesArchiveComponentPath] + ) + ); + + $excludes = $this->getRegexpsForPaths([ + 'css', + 'js', + 'styles', + 'php', + ]); + + $this->filesystem->mirror( + $drupalFilesPath, + $drupalFilesArchiveComponentPath, + $this->getFileIterator($drupalFilesPath, $excludes) + ); + + return $drupalFilesArchiveComponentPath; + } + + /** + * Returns the path to Drupal files directory. + * + * @return string + * + * @throws \Exception + */ + private function getDrupalFilesDir(): string + { + if (isset($this->drupalFilesDir)) { + return $this->drupalFilesDir; + } + + Drush::bootstrapManager()->doBootstrap(DrupalBootLevels::FULL); + $drupalFilesPath = Drupal::service('file_system')->realpath('public://'); + if (!$drupalFilesPath) { + throw new Exception(dt('Path to Drupal files is empty.')); + } + + return $this->drupalFilesDir = $drupalFilesPath; + } + + /** + * Returns file iterator. + * + * Excludes paths according to the list of excludes provides. + * Validates for sensitive data present. + * + * @param string $path + * Directory. + * @param array $excludes + * The list of file exclude rules (regular expressions). + * + * @return \Traversable + */ + private function getFileIterator(string $path, array $excludes): Traversable + { + return new RecursiveIteratorIterator( + new RecursiveCallbackFilterIterator( + new RecursiveDirectoryIterator( + $path, + FilesystemIterator::SKIP_DOTS + ), + function ($file) use ($excludes, $path) { + $localFileName = str_replace($path, '', $file); + $localFileName = str_replace('\\', '/', $localFileName); + $localFileName = trim($localFileName, '\/'); + + foreach ($excludes as $exclude) { + if (preg_match($exclude, $localFileName)) { + $this->logger()->info(dt( + 'Path excluded (!exclude): !path', + ['!exclude' => $exclude, '!path' => $localFileName] + )); + + return false; + } + } + + $this->validateSensitiveData($file, $localFileName); + + return true; + } + ) + ); + } + + /** + * Creates "database" archive component and returns the absolute path. + * + * @param array $options + * The command options. + * + * @return string + * The full path to the database archive component directory. + * + * @throws \Exception + * + * @see \Drush\Commands\sql\SqlCommands::dump() + */ + private function getDatabaseComponentPath(array $options): string + { + $this->logger()->info(dt('Creating database SQL dump file...')); + $databaseArchiveDir = Path::join($this->archiveDir, self::COMPONENT_DATABASE); + $this->filesystem->mkdir($databaseArchiveDir); + + $options['result-file'] = Path::join($databaseArchiveDir, self::SQL_DUMP_FILE_NAME); + $sql = SqlBase::create($options); + if (false === $sql->dump()) { + throw new Exception(dt('Unable to dump database. Rerun with --debug to see any error message.')); + } + + return $databaseArchiveDir; + } + + /** + * Returns the list of regular expressions to match paths. + * + * @param array $paths + * The list of paths to match. + * + * @return array + */ + private function getRegexpsForPaths(array $paths): array + { + return array_map( + fn($path) => sprintf('#^%s$#', trim($path)), + $paths + ); + } + + /** + * Returns docroot directory name with trailing escaped slash for a "web" docroot site for use in regular expressions, otherwise - empty string. + * + * @return string + * + * @throws \Exception + */ + private function getDocrootRegexpPrefix(): string + { + return $this->isWebRootSite() ? basename($this->getRoot()) . '/' : ''; + } + + /** + * Returns the list of regular expressions to match Drupal files paths and sites/@/settings.@.php files. + * + * @return array + * + * @throws \Exception + */ + private function getDrupalExcludes(): array + { + $excludes = [ + '#^' . $this->getDocrootRegexpPrefix() . 'sites/.+/settings\..+\.php$#', + ]; + + $drupalFilesPath = $this->getDrupalFilesDir(); + $drupalFilesPathRelative = Path::makeRelative($drupalFilesPath, $this->getComposerRoot()); + $excludes[] = '#^' . $drupalFilesPathRelative . '$#'; + + return $excludes; + } + + /** + * Validates files for sensitive data (database connection). + * + * Prevents creating a code archive containing a [docroot]/sites/@/settings.php file with database connection settings + * defined. + * + * @param string $file + * The absolute path to the file. + * @param string $localFileName + * The local (project-base) path to the file. + * + * @throws \Exception + */ + private function validateSensitiveData(string $file, string $localFileName): void + { + $regexp = '#^' . $this->getDocrootRegexpPrefix() . 'sites/.*/settings\.php$#'; + if (!preg_match($regexp, $localFileName)) { + return; + } + + $settingsPhpFileContents = file_get_contents($file); + $settingsWithoutComments = preg_replace('/\/\*(.*?)\*\/|(\/\/|#)(.*?)$/ms', '', $settingsPhpFileContents); + $isDatabaseSettingsPresent = preg_match('/\$databases[^;]*=[^;]*(\[|(array[^;]*\())[^;]+(\]|\))[^;]*;/ms', $settingsWithoutComments); + if ($isDatabaseSettingsPresent) { + throw new Exception( + dt( + 'Found database connection settings in !path. It is risky to include them to the archive. Please move the database connection settings into a setting.*.php file or exclude them from the archive with "--exclude-code-paths=!path".', + ['!path' => $localFileName] + ) + ); + } + } + + /** + * Provides basic verification/correction on destination option. + * + * @param string $destination + * + * @return void + */ + private function destinationCleanup($destination) + { + // User input may be in the wrong format, this performs some basic + // corrections. The correct format should include a .tar.gz. + if (substr($destination, -7) !== ".tar.gz") { + // If the user provided .tar but not .gz. + if (substr($destination, -4) === ".tar") { + return $destination . ".gz"; + } + + // If neither, the user provided a directory. + if (substr($destination, -1) === "/") { + return $destination . "archive.tar.gz"; + } else { + return $destination . "/archive.tar.gz"; + } + } + return $destination; + } +} diff --git a/src/Commands/core/ArchiveRestoreCommands.php b/src/Commands/core/ArchiveRestoreCommands.php new file mode 100644 index 0000000000..fd9e7e320b --- /dev/null +++ b/src/Commands/core/ArchiveRestoreCommands.php @@ -0,0 +1,716 @@ + null, + 'overwrite' => false, + 'site-subdir' => self::SITE_SUBDIR, + 'setup-database-connection' => true, + 'code' => false, + 'code-source-path' => null, + 'files' => false, + 'files-source-path' => null, + 'files-destination-relative-path' => null, + 'db' => false, + 'db-source-path' => null, + 'db-driver' => 'mysql', + 'db-port' => null, + 'db-host' => null, + 'db-name' => null, + 'db-user' => null, + 'db-password' => null, + 'db-prefix' => null, + ] + ): void { + $siteAlias = $this->getSiteAlias($site); + if (!$siteAlias->isLocal()) { + throw new Exception( + dt( + 'Could not restore archive !path into site !site: restoring an archive into a local site is not supported.', + ['!path' => $path, '!site' => $site] + ) + ); + } + + if (!$options['code'] && !$options['files'] && !$options['db']) { + $options['code'] = $options['files'] = $options['db'] = true; + } + + if (($options['code'] || $options['files']) && !self::programExists('rsync')) { + throw new Exception( + dt('Could not restore the code or the Drupal files: "rsync" program not found') + ); + } + + $this->filesystem = new Filesystem(); + $extractDir = $this->getExtractDir($path); + + foreach (['code' => 'code', 'db' => 'database', 'files' => 'files'] as $component => $label) { + if (!$options[$component]) { + continue; + } + + // Validate requested components have sources. + if (null === $extractDir && null === $options[$component . '-source-path']) { + throw new Exception( + dt( + 'Missing either "path" input or "!component_path" option for the !label component.', + [ + '!component' => $component, + '!label' => $label, + ] + ) + ); + } + } + + if ($options['destination-path']) { + $this->destinationPath = $options['destination-path']; + } + + // If the destination path was not specified, extract over the current site + if (!$this->destinationPath) { + $bootstrapManager = Drush::bootstrapManager(); + $this->destinationPath = $bootstrapManager->getComposerRoot(); + } + + // If there isn't a current site either, then extract to the cwd + if (!$this->destinationPath) { + $siteDirName = basename(basename($path, '.tgz'), 'tar.gz'); + $this->destinationPath = Path::join(getcwd(), $siteDirName); + } + + if ($options['code'] && is_dir($this->destinationPath)) { + if (!$options['overwrite']) { + throw new Exception( + dt('Destination path !path already exists (use "--overwrite" option).', ['!path' => $this->destinationPath]) + ); + } + + if ( + !$this->io()->confirm( + dt( + 'Destination path !path already exists. Are you sure you want to delete !path directory before restoring the archive into it?', + [ + '!path' => $this->destinationPath, + ] + ) + ) + ) { + throw new UserAbortException(); + } + + // Remove destination if --overwrite option is set. + $this->filesystem->remove($this->destinationPath); + } + + // Create the destination if it does not already exist + if (!is_dir($this->destinationPath) && !mkdir($this->destinationPath)) { + throw new Exception(dt('Failed creating destination directory "!destination"', ['!destination' => $this->destinationPath])); + } + + $this->destinationPath = realpath($this->destinationPath); + + if ($options['code']) { + $codeComponentPath = $options['code-source-path'] ?? Path::join($extractDir, self::COMPONENT_CODE); + $this->importCode($codeComponentPath); + } + + if ($options['files']) { + $filesComponentPath = $options['files-source-path'] ?? Path::join($extractDir, self::COMPONENT_FILES); + $this->importFiles($filesComponentPath, $options); + } + + if ($options['db']) { + $databaseComponentPath = $options['db-source-path'] ?? Path::join($extractDir, self::COMPONENT_DATABASE, self::SQL_DUMP_FILE_NAME); + $this->importDatabase($databaseComponentPath, $options); + } + + $this->logger()->info(dt('Done!')); + } + + /** + * Extracts the archive. + * + * @param string|null $path + * The path to the archive file. + * + * @return string|null + * + * @throws \Exception + */ + protected function getExtractDir(?string $path): ?string + { + if (null === $path) { + return null; + } + + if (is_dir($path)) { + return $path; + } + + $this->logger()->info('Extracting the archive...'); + + if (!is_file($path)) { + throw new Exception(dt('File !path is not found.', ['!path' => $path])); + } + + if (!preg_match('/\.tar\.gz$/', $path) && !preg_match('/\.tgz$/', $path)) { + throw new Exception(dt('File !path is not a *.tar.gz file.', ['!path' => $path])); + } + + ['filename' => $archiveFileName] = pathinfo($path); + $archiveFileName = str_replace('.tar', '', $archiveFileName); + + $extractDir = Path::join(FsUtils::tmpDir(), $archiveFileName); + $this->filesystem->mkdir($extractDir); + + $archive = new PharData($path); + $archive->extractTo($extractDir); + + $this->logger()->info(dt('The archive successfully extracted into !path', ['!path' => $extractDir])); + + return $extractDir; + } + + /** + * Imports the code to the site. + * + * @param string $source + * The path to the code files directory. + * + * @throws \Exception + */ + protected function importCode(string $source): void + { + $this->logger()->info('Importing code...'); + + if (!is_dir($source)) { + throw new Exception(dt('Directory !path not found.', ['!path' => $source])); + } + + $this->rsyncFiles($source, $this->getDestinationPath()); + + $composerJsonPath = Path::join($this->getDestinationPath(), 'composer.json'); + if (is_file($composerJsonPath)) { + $this->logger()->success( + dt('composer.json is found (!path), install Composer dependencies with composer install.'), + ['!path' => $composerJsonPath] + ); + } + } + + /** + * Imports Drupal files to the site. + * + * @param string $source + * The path to the source directory. + * @param array $options + * The options. + * + * @throws \Exception + */ + protected function importFiles(string $source, array $options): void + { + $this->logger()->info('Importing files...'); + + if (!is_dir($source)) { + throw new Exception(dt('The source directory !path not found for files.', ['!path' => $source])); + } + + $destinationAbsolute = $this->fileImportAbsolutePath($options['files-destination-relative-path']); + + if ( + is_dir($destinationAbsolute) && + (!$options['code'] || !$options['overwrite']) && + !$this->io()->confirm( + dt( + 'Destination Drupal files path !path already exists. Are you sure you want restore Drupal files archive into it?', + [ + '!path' => $destinationAbsolute, + ] + ) + ) + ) { + throw new UserAbortException(); + } + + $this->filesystem->mkdir($destinationAbsolute); + $this->rsyncFiles($source, $destinationAbsolute); + } + + /** + * Determines the path where files should be extracted. + * + * @param null|string $destinationRelative + * The relative path to the Drupal files directory. + * + * @return string + * The absolute path to the Drupal files directory. + * + * @throws \Exception + */ + protected function fileImportAbsolutePath(?string $destinationRelative): string + { + // If the user specified the path to the files directory, use that. + if ($destinationRelative) { + return Path::join($this->getDestinationPath(), $destinationRelative); + } + + // If we are extracting over an existing site, query Drupal to get the files path + $bootstrapManager = Drush::bootstrapManager(); + $path = $bootstrapManager->getComposerRoot(); + if (!empty($path)) { + try { + $bootstrapManager->doBootstrap(DrupalBootLevels::FULL); + return Drupal::service('file_system')->realpath('public://'); + } catch (Throwable $t) { + $this->logger()->warning('Could not bootstrap Drupal site at destination to determine file path'); + } + } + + // Find the Drupal root for the archived code, and assume sites/default/files. + $drupalRootPath = $this->getDrupalRootPath(); + if ($drupalRootPath) { + return Path::join($drupalRootPath, 'sites/default/files'); + } + + throw new Exception( + dt( + 'Can\'t detect relative path for Drupal files for destination "!destination": missing --files-destination-relative-path option.', + ['!destination' => $this->getDestinationPath()] + ) + ); + } + + /** + * Returns the absolute path to Drupal root. + * + * @return string|null + */ + protected function getDrupalRootPath(): ?string + { + $composerRoot = $this->getDestinationPath(); + $drupalFinder = new DrupalFinder(); + if (!$drupalFinder->locateRoot($composerRoot)) { + return null; + } + + return $drupalFinder->getDrupalRoot(); + } + + /** + * Returns the destination path. + * + * @return string + */ + protected function getDestinationPath(): string + { + return $this->destinationPath; + } + + /** + * Returns SiteAlias object by the site alias name. + * + * @param string|null $site + * The site alias. + * + * @return \Consolidation\SiteAlias\SiteAlias + * + * @throws \Exception + */ + protected function getSiteAlias(?string $site): SiteAlias + { + $pathEvaluator = new BackendPathEvaluator(); + /** @var \Consolidation\SiteAlias\SiteAliasManager $manager */ + $manager = $this->siteAliasManager(); + + if (null !== $site) { + $site .= ':%root'; + } + $evaluatedPath = HostPath::create($manager, $site); + $pathEvaluator->evaluate($evaluatedPath); + + return $evaluatedPath->getSiteAlias(); + } + + /** + * Copies files from the source to the destination. + * + * @param string $source + * The source path. + * @param string $destination + * The destination path. + * + * @throws \Exception + */ + protected function rsyncFiles(string $source, string $destination): void + { + $source = rtrim($source, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR; + $destination = rtrim($destination, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR; + + if ( + !$this->io()->confirm( + dt( + 'Are you sure you want to sync files from "!source" to "!destination"?', + [ + '!source' => $source, + '!destination' => $destination, + ] + ) + ) + ) { + throw new UserAbortException(); + } + + if (!is_dir($source)) { + throw new Exception(dt('The source directory !path not found.', ['!path' => $source])); + } + + $this->logger()->info( + dt( + 'Copying files from "!source" to "!destination"...', + [ + '!source' => $source, + '!destination' => $destination, + ] + ) + ); + + $options[] = '-akz'; + if ($this->output()->isVerbose()) { + $options[] = '--stats'; + $options[] = '--progress'; + $options[] = '-v'; + } + + $command = sprintf( + 'rsync %s %s %s', + implode(' ', $options), + $source, + $destination + ); + + /** @var \Consolidation\SiteProcess\ProcessBase $process */ + $process = $this->processManager()->shell($command); + $process->run($process->showRealtime()); + if ($process->isSuccessful()) { + return; + } + + throw new Exception( + dt( + 'Failed to copy files from !source to !destination: !error', + [ + '!source' => $source, + '!destination' => $destination, + '!error' => $process->getErrorOutput(), + ] + ) + ); + } + + /** + * Imports the database dump to the site. + * + * @param string $databaseDumpPath + * The path to the database dump file. + * @param array $options + * The command options. + * + * @throws \Drush\Exceptions\UserAbortException + * @throws \Exception + */ + protected function importDatabase(string $databaseDumpPath, array $options): void + { + $this->logger()->info('Importing database...'); + + if (!is_file($databaseDumpPath)) { + throw new Exception(dt('Database dump file !path not found.', ['!path' => $databaseDumpPath])); + } + + $sqlOptions = []; + if (isset($options['db-url'])) { + $sqlOptions = ['db-url' => $options['db-url']]; + } else if ($options['db-name']) { + $connection = [ + 'driver' => $options['db-driver'], + 'port' => $options['db-port'], + 'prefix' => $options['db-prefix'], + 'host' => $options['db-host'], + 'database' => $options['db-name'], + 'username' => $options['db-user'], + 'password' => $options['db-password'], + ]; + + $sqlOptions = [ + 'databases' => [ + 'default' => [ + 'default' => $connection, + ], + ], + ]; + } else if ($options['destination-path']) { + throw new Exception('Database connection settings are required if --destination-path option is provided'); + } else { + $bootstrapManager = Drush::bootstrapManager(); + $bootstrapManager->doBootstrap(DrupalBootLevels::CONFIGURATION); + } + + try { + $sql = SqlBase::create($sqlOptions); + $isDbExist = $sql->dbExists(); + $databaseSpec = $sql->getDbSpec(); + } catch (Throwable $t) { + throw new Exception(dt('Failed to get database specification: !error', ['!error' => $t->getMessage()])); + } + + if ( + $isDbExist && + !$this->io()->confirm( + dt( + 'Are you sure you want to drop the database "!database" (username: !user, password: !password, port: !port, prefix: !prefix) and import the database dump "!path"?', + [ + '!path' => $databaseDumpPath, + '!database' => $databaseSpec['database'], + '!user' => $databaseSpec['username'], + '!password' => isset($databaseSpec['password']) ? '******' : '[not set]', + '!port' => $databaseSpec['port'] ?: dt('n/a'), + '!prefix' => $databaseSpec['prefix'] ?: dt('n/a'), + ] + ) + ) + ) { + throw new UserAbortException(); + } + + if ($isDbExist && !$sql->drop($sql->listTablesQuoted())) { + throw new Exception( + dt('Failed to drop database !database.', ['!database' => $databaseSpec['database']]) + ); + } elseif (!$sql->createdb(true)) { + throw new Exception( + dt('Failed to create database !database.', ['!database' => $databaseSpec['database']]) + ); + } + + $sql->setDbSpec($databaseSpec); + if (!$sql->query('', $databaseDumpPath)) { + throw new Exception(dt('Database import has failed: !error', ['!error' => $sql->getProcess()->getErrorOutput()])); + } + + if ($sqlOptions) { + // Setup settings.local.php file since database connection settings provided via options. + $this->setupLocalSettingsPhp($databaseSpec, $options); + } + } + + /** + * Sets up settings.local.php file. + * + * 1. Creates settings.php file (a copy of default.settings.php) in the site's subdirectory if not exists; + * 2. Makes sure settings.php has an active (i.e. uncommented) "include settings.local.php file" directive; + * 3. Updates settings.local.php file to include database connection settings provided via command's options. + * + * @param array $databaseSpec + * The database connection specification. + * @param array $options + * The command options. + * + * @throws Exception + */ + private function setupLocalSettingsPhp(array $databaseSpec, array $options): void + { + $drupalRootPath = $this->getDrupalRootPath(); + if (!$drupalRootPath) { + throw new Exception( + dt('Failed to detect Drupal docroot path for path !path', ['!path' => $this->getDestinationPath()]) + ); + } + + $siteSubdir = Path::join($drupalRootPath, 'sites', $options['site-subdir']); + $this->filesystem->mkdir($siteSubdir); + + $settingsPhpPath = Path::join($siteSubdir, 'settings.php'); + if (!is_file($settingsPhpPath)) { + // Create settings.php file as a copy of default.settings.php file. + $defaultSettingsPath = Path::join($drupalRootPath, 'sites', self::SITE_SUBDIR, 'default.settings.php'); + $this->logger()->info('Copying !from to !to...', ['!from' => $defaultSettingsPath, '!to' => $settingsPhpPath]); + copy( + $defaultSettingsPath, + $settingsPhpPath + ); + } + + $drushSignature = '// Added by Drush archive:restore command.'; + + // Make sure settings.php has an active (i.e. uncommented) "include settings.local.php file" directive. + $settingsPhpContent = file_get_contents($settingsPhpPath); + if (preg_match('/\# if \(file_exists.+?settings\.local\.php.+?\# }/ms', $settingsPhpContent, $matches)) { + $uncommentedLocalSettingsInclude = $drushSignature . "\n" . str_replace('# ', '', $matches[0]); + + $settingsPhpIncludeLocalContent = str_replace( + $matches[0], + $uncommentedLocalSettingsInclude, + $settingsPhpContent + ); + + $this->logger()->info(sprintf('Updating %s to include settings.local.php file...', $settingsPhpPath)); + if (!file_put_contents($settingsPhpPath, $settingsPhpIncludeLocalContent)) { + throw new Exception(dt('Failed to save updated !path', ['!path' => $settingsPhpPath])); + } + } + + $databaseSpecExported = var_export($databaseSpec, true); + $settingsLocalPhpPath = Path::join($siteSubdir, 'settings.local.php'); + $settingsLocalPhpDatabaseConnection = <<logger()->info('Creating !path with database connection settings...', ['!path' => $settingsLocalPhpPath]); + $settingsLocalPhpModifiedContent = 'saveSettingsLocalPhp($settingsLocalPhpPath, $settingsLocalPhpModifiedContent); + return; + } + + $settingsLocalPhpContent = file_get_contents($settingsLocalPhpPath); + if (false === strpos($settingsLocalPhpContent, $drushSignature)) { + $this->logger()->info('Adding database connection settings to !path...', ['!path' => $settingsLocalPhpPath]); + $settingsLocalPhpModifiedContent = $settingsLocalPhpContent . $settingsLocalPhpDatabaseConnection; + $this->saveSettingsLocalPhp($settingsLocalPhpPath, $settingsLocalPhpModifiedContent); + return; + } + + $this->logger()->info('Updating database connection settings in !path...', ['!path' => $settingsLocalPhpPath]); + $settingsLocalPhpModifiedContent = preg_replace( + '/' . preg_quote($drushSignature, '/') . '.+?\);/ms', + $settingsLocalPhpDatabaseConnection, + $settingsLocalPhpContent + ); + $this->saveSettingsLocalPhp($settingsLocalPhpPath, $settingsLocalPhpModifiedContent); + } + + /** + * Saves settings.local.php file with actual database connection settings. + * + * @param string $path + * @param string $content + * @throws Exception + */ + private function saveSettingsLocalPhp(string $path, string $content): void + { + if (!file_put_contents($path, $content)) { + throw new Exception(dt('Failed to create or update !path.', ['!path' => $path])); + } + } +} diff --git a/src/Utils/FsUtils.php b/src/Utils/FsUtils.php index c093d49e6d..fdaf90d133 100644 --- a/src/Utils/FsUtils.php +++ b/src/Utils/FsUtils.php @@ -10,6 +10,9 @@ class FsUtils { + // @var null|string[] List of directories to delete + private static $deletionList = null; + /** * Decide where our backup directory should go * @@ -113,11 +116,66 @@ public static function isUsableDirectory(?string $dir) } /** - * Prepare a backup directory. + * Prepare a temporary directory that will be deleted on exit. * * @param string $subdir * A string naming the subdirectory of the backup directory. + * @return string + * Path to the specified backup directory. + * @throws \Exception + */ + public static function tmpDir($subdir = null): string + { + $parent = self::getBackupDirParent(); + $fs = new Filesystem(); + $dir = $fs->tempnam($parent, $subdir ?? 'drush'); + unlink($dir); + $fs->mkdir($dir); + static::registerForDeletion($dir); + return $dir; + } + + /** + * Add the given directory to a list to be deleted on exit. * + * @param string $dir + * Path to directory to be deleted later. + */ + public static function registerForDeletion(string $dir) + { + if (!isset(static::$deletionList)) { + static::$deletionList = []; + register_shutdown_function([static::class, 'cleanup']); + } + + static::$deletionList[] = $dir; + } + + /** + * Delete all of the files registered for deletion. + */ + public static function cleanup() + { + if (!isset(static::$deletionList)) { + return; + } + + $fs = new Filesystem(); + foreach (static::$deletionList as $dir) { + try { + $fs->remove($dir); + } catch (IOException $e) { + // No action taken if someone already deleted the directory + } + } + } + + /** + * Prepare a backup directory. + * + * @param string $subdir + * A string naming the subdirectory of the backup directory. + * @return string * Path to the specified backup directory. * @throws \Exception */ @@ -137,7 +195,7 @@ public static function prepareBackupDir($subdir = null): string * * @param string $path * The path being checked. - * + * @return string * The canonicalized absolute pathname. */ public static function realpath(string $path): string diff --git a/tests/functional/ArchiveTest.php b/tests/functional/ArchiveTest.php new file mode 100644 index 0000000000..08eb9834d2 --- /dev/null +++ b/tests/functional/ArchiveTest.php @@ -0,0 +1,366 @@ +setUpDrupal(1, true); + $this->archiveDumpOptions = [ + 'db' => null, + 'files' => null, + 'code' => null, + 'exclude-code-paths' => 'sut/sites/.+/settings.php,(?!sut|composer\.json|composer\.lock).*', + ]; + + $this->archivePath = Path::join($this->getSandbox(), 'archive.tar.gz'); + $this->drush( + 'archive:dump', + [], + array_merge($this->archiveDumpOptions, [ + 'destination' => $this->archivePath, + 'overwrite' => null, + ]) + ); + $actualArchivePath = $this->getOutput(); + $this->assertEquals($this->archivePath, $actualArchivePath); + + $this->restorePath = Path::join($this->getSandbox(), 'restore'); + $this->removeDir($this->restorePath); + + $this->extractPath = Path::join($this->getSandbox(), 'extract'); + $this->removeDir($this->extractPath); + $archive = new PharData($this->archivePath); + $archive->extractTo($this->extractPath); + + $this->drush( + 'status', + [], + ['format' => 'json'] + ); + $this->fixtureDatabaseSettings = json_decode($this->getOutput(), true); + $this->fixtureDatabaseSettings['db-name'] = 'archive_dump_restore_test_' . mt_rand(); + $dbUrlParts = explode(':', self::getDbUrl()); + $this->fixtureDatabaseSettings['db-password'] = substr($dbUrlParts[2], 0, strpos($dbUrlParts[2], '@')); + $fixtureDbUrl = self::getDbUrl() . '/' . $this->fixtureDatabaseSettings['db-name']; + + $this->archiveRestoreOptions = [ + 'destination-path' => $this->restorePath, + 'overwrite' => null, + 'site-subdir' => 'dev', + 'db-url' => $fixtureDbUrl, + ]; + } + + public function testArchiveDumpCommand(): void + { + // Create a file at the destination to confirm that archive:dump + // will fail without --overwrite in this instance. + file_put_contents($this->archivePath, "Existing file at destination"); + + // Try to overwrite the existing archive with "--destination". + $this->drush( + 'archive:dump', + [], + array_merge($this->archiveDumpOptions, [ + 'destination' => $this->archivePath, + ]), + null, + null, + self::EXIT_ERROR + ); + $this->assertStringContainsString('The destination file already exists.', $this->getErrorOutput()); + + // Overwrite the existing archive with "--destination" and "--override". + $this->drush( + 'archive:dump', + [], + array_merge($this->archiveDumpOptions, [ + 'destination' => $this->archivePath, + 'overwrite' => null, + ]) + ); + $actualArchivePath = $this->getOutput(); + $this->assertEquals($this->archivePath, $actualArchivePath); + + // Validate database credentials are present in settings.php file. + $this->drush( + 'archive:dump', + [], + [], + null, + null, + self::EXIT_ERROR + ); + $this->assertStringContainsString( + 'Found database connection settings', + $this->getErrorOutput() + ); + } + + public function testArchiveRestoreCommand(): void + { + // [info] Copying files from "C:/projects/work/sandbox/archive/code\" to "C:\projects\work\"... + // [info] Executing: rsync -akz --stats --progress -v C:/projects/work/sandbox/archive/code\ C:\projects\work\ + // > The source and destination cannot both be remote. + if ($this->isWindows()) { + $this->markTestSkipped('The command archive:restore does not work on Windows yet due to an rsync issue.'); + } + + // Restoring to sqlite fails + if ($this->dbDriver() === 'sqlite') { + $this->markTestSkipped('The command archive:restore cannot restore to an sqlite database.'); + } + + // Restore the code from a source path. + $testFileName = 'test-file-' . mt_rand() . '.txt'; + file_put_contents(Path::join($this->extractPath, 'code', 'sut', $testFileName), 'foo_bar'); + $this->drush( + 'archive:restore', + [], + array_merge($this->archiveRestoreOptions, [ + 'code' => null, + 'code-source-path' => Path::join($this->extractPath, 'code'), + ]) + ); + $this->assertTrue(is_file(Path::join($this->restorePath, 'sut', $testFileName))); + $this->assertTrue(is_file(Path::join($this->restorePath, 'composer.json'))); + $this->assertTrue(!is_dir(Path::join($this->restorePath, 'vendor'))); + $this->assertTrue(!is_file(Path::join($this->restorePath, 'sut', 'sites', 'dev', 'settings.php'))); + $this->assertTrue(!is_file(Path::join($this->restorePath, 'sut', 'sites', 'dev', 'settings.local.php'))); + + // Restore archive from an existing file and an existing destination path. + $this->drush( + 'archive:restore', + [$this->archivePath], + array_diff_key($this->archiveRestoreOptions, ['overwrite' => null]), + null, + null, + self::EXIT_ERROR + ); + $this->assertMatchesRegularExpression( + '/Destination path .+ already exists/', + str_replace("\n", " ", $this->getErrorOutput()) + ); + + // Restore archive from an existing file. + $this->drush( + 'archive:restore', + [$this->archivePath], + $this->archiveRestoreOptions + ); + $this->installComposerDependencies(); + $this->assertRestoredSiteStatus(); + + // Restore the Drupal files from a source path. + file_put_contents(Path::join($this->extractPath, 'files', $testFileName), 'foo_bar'); + $filesRelativePath = Path::join('sut', 'sites', 'default', 'files'); + $this->drush( + 'archive:restore', + [], + array_merge($this->archiveRestoreOptions, [ + 'files' => null, + 'files-source-path' => Path::join($this->extractPath, 'files'), + 'files-destination-relative-path' => $filesRelativePath, + ]) + ); + $this->assertTrue(is_file(Path::join($this->restorePath, $filesRelativePath, $testFileName))); + $this->assertRestoredSiteStatus(); + + // Restore the database from a source path. + $this->drush( + 'archive:restore', + [], + array_merge($this->archiveRestoreOptions, [ + 'db' => null, + 'db-source-path' => Path::join($this->extractPath, 'database', 'database.sql'), + ]) + ); + $this->assertRestoredSiteStatus(); + + // Restore database with invalid --db-url. + $this->drush( + 'archive:restore', + [], + array_merge($this->archiveRestoreOptions, [ + 'db' => null, + 'db-source-path' => Path::join($this->extractPath, 'database', 'database.sql'), + 'db-url' => 'bad://db@url/schema', + ]), + null, + null, + self::EXIT_ERROR + ); + $this->assertStringContainsString( + 'Failed to get database specification:', + $this->getErrorOutput() + ); + $this->assertRestoredSiteStatus(); + + // Restore database with --db-url option with an invalid host. + $this->drush( + 'archive:restore', + [], + array_merge($this->archiveRestoreOptions, [ + 'db' => null, + 'db-source-path' => Path::join($this->extractPath, 'database', 'database.sql'), + 'db-url' => sprintf( + '%s://%s:%s@%s/%s', + $this->fixtureDatabaseSettings['db-driver'], + $this->fixtureDatabaseSettings['db-username'], + $this->fixtureDatabaseSettings['db-password'], + 'invalid_host', + $this->fixtureDatabaseSettings['db-name'] + ), + ]), + null, + null, + self::EXIT_ERROR + ); + $this->assertStringContainsString( + sprintf('Failed to create database %s.', $this->fixtureDatabaseSettings['db-name']), + $this->getErrorOutput() + ); + $this->assertRestoredSiteStatus(); + + // Restore database with a set of database connection options. + $this->drush( + 'archive:restore', + [], + array_merge( + array_diff_key($this->archiveRestoreOptions, ['db-url' => null]), + [ + 'db' => null, + 'db-source-path' => Path::join($this->extractPath, 'database', 'database.sql'), + 'db-driver' => $this->fixtureDatabaseSettings['db-driver'], + 'db-name' => $this->fixtureDatabaseSettings['db-name'], + 'db-host' => $this->fixtureDatabaseSettings['db-hostname'], + 'db-user' => $this->fixtureDatabaseSettings['db-username'], + 'db-password' => $this->fixtureDatabaseSettings['db-password'], + ] + ) + ); + $this->assertRestoredSiteStatus(); + + // Restore database with a set of database connection options with an invalid host. + $this->drush( + 'archive:restore', + [], + array_merge( + array_diff_key($this->archiveRestoreOptions, ['db-url' => null]), + [ + 'db' => null, + 'db-source-path' => Path::join($this->extractPath, 'database', 'database.sql'), + 'db-driver' => $this->fixtureDatabaseSettings['db-driver'], + 'db-name' => $this->fixtureDatabaseSettings['db-name'], + 'db-host' => 'invalid_host', + 'db-user' => $this->fixtureDatabaseSettings['db-username'], + 'db-password' => $this->fixtureDatabaseSettings['db-password'], + ] + ), + null, + null, + self::EXIT_ERROR + ); + $this->assertStringContainsString( + sprintf('Failed to create database %s.', $this->fixtureDatabaseSettings['db-name']), + $this->getErrorOutput() + ); + $this->assertRestoredSiteStatus(); + + // Restore archive from a non-existing file. + $nonExistingArchivePath = Path::join($this->getSandbox(), 'arch.tar.gz'); + $this->drush( + 'archive:restore', + [$nonExistingArchivePath], + $this->archiveRestoreOptions, + null, + null, + self::EXIT_ERROR + ); + $this->assertStringContainsString( + 'arch.tar.gz is not found', + $this->getErrorOutput() + ); + $this->assertRestoredSiteStatus(); + + // Restore database without database connection settings. + $this->drush( + 'archive:restore', + [], + array_merge( + array_diff_key($this->archiveRestoreOptions, ['db-url' => null]), + [ + 'db' => null, + 'db-source-path' => Path::join($this->extractPath, 'database', 'database.sql'), + ] + ), + null, + null, + self::EXIT_ERROR + ); + $this->assertStringContainsString( + 'Database connection settings are required if --destination-path', + $this->getErrorOutput() + ); + $this->assertRestoredSiteStatus(); + } + + /** + * Asserts the status of restored site. + */ + private function assertRestoredSiteStatus(): void + { + $this->assertTrue(is_file(Path::join($this->restorePath, 'composer.json'))); + $this->assertTrue(is_file(Path::join($this->restorePath, 'sut', 'sites', 'dev', 'settings.php'))); + $this->assertTrue(is_file(Path::join($this->restorePath, 'sut', 'sites', 'dev', 'settings.local.php'))); + + $this->drush( + 'status', + [], + ['format' => 'json'], + null, + Path::join($this->restorePath, 'sut') + ); + $restoredSiteStatus = json_decode($this->getOutput(), true); + $this->assertEquals('Connected', $restoredSiteStatus['db-status']); + $this->assertEquals(Path::join($this->restorePath, 'sut'), $restoredSiteStatus['root']); + $this->assertEquals($this->fixtureDatabaseSettings['db-name'], $restoredSiteStatus['db-name']); + } + + /** + * Executes `composer install` in the restored site's composer root. + */ + private function installComposerDependencies(): void + { + $process = new Process(['composer', 'install'], $this->restorePath, null, null, 180); + $process->run(); + $this->assertTrue( + $process->isSuccessful(), + sprintf('"composer install" has failed: %s', $process->getErrorOutput()) + ); + } +} diff --git a/tests/unish/Utils/FSUtils.php b/tests/unish/Utils/FSUtils.php index eac6df48d3..930eb8627c 100644 --- a/tests/unish/Utils/FSUtils.php +++ b/tests/unish/Utils/FSUtils.php @@ -7,6 +7,10 @@ trait FSUtils { public function removeDir($dir) { + if (!is_dir($dir)) { + return; + } + $files = array_diff(scandir($dir), ['.','..']); foreach ($files as $file) { if (is_dir("$dir/$file") && !is_link("$dir/$file")) {