From 2cfca2b59fb956a5f3a0cdc5ef0d9235b0140b46 Mon Sep 17 00:00:00 2001 From: Rich Gerdes Date: Mon, 18 Jun 2018 16:49:20 -0400 Subject: [PATCH 1/4] Factor out gitlab configuration from command into reusable base class. --- .../SatisGitlab/Command/GitlabCommandBase.php | 198 ++++++++++++++++++ .../Command/GitlabToConfigCommand.php | 186 +++------------- 2 files changed, 226 insertions(+), 158 deletions(-) create mode 100644 src/MBO/SatisGitlab/Command/GitlabCommandBase.php diff --git a/src/MBO/SatisGitlab/Command/GitlabCommandBase.php b/src/MBO/SatisGitlab/Command/GitlabCommandBase.php new file mode 100644 index 0000000..24b5635 --- /dev/null +++ b/src/MBO/SatisGitlab/Command/GitlabCommandBase.php @@ -0,0 +1,198 @@ +setDescription('populate satis required packages by scanning gitlab repositories') + ->setHelp('look for composer.json in default gitlab branche, extract dependencies and register them in SATIS configuration') + ->addArgument('gitlab-url', InputArgument::OPTIONAL, 'gitlab instance url', static::DEFAULT_VALUE) + ->addArgument('gitlab-token') + + // deep customization : template file extended with default configuration + ->addOption('template', null, InputOption::VALUE_REQUIRED, 'template satis.json extended with gitlab repositories', $templatePath) + + ->addOption('no-token', null, InputOption::VALUE_NONE, 'disable token writing in output configuration') + + // output configuration + ->addOption('output', 'O', InputOption::VALUE_REQUIRED, 'output config file', 'satis.json') + ; + } + + protected function projectName(array $project, array $composer) { + $project_name = isset($composer['name']) ? $composer['name'] : null; + if (is_null($project_name)) { + // User project path as name if composer.json does not have one. + $project_name = $project['path_with_namespace']; + $count_slashes = substr_count($project_name, '/'); + if ($count_slashes > 0) { + // if there is more then one slash, replace all but last + $project_name = str_replace ('/', $project_name, '-', $count_slashes - 1); + } + } + return $project_name; + } + + protected function processProject(InputInterface $input, OutputInterface $output, array &$satis, array $project, array $composer) { + // Function not required by default. + } + + protected function processSatisConfiguration(InputInterface $input, OutputInterface $output, array &$satis) { + // Function not required by default. + } + + protected function execute(InputInterface $input, OutputInterface $output) { + + /* + * load template satis.json file + */ + $templatePath = $input->getOption('template'); + $output->writeln(sprintf("Loading template %s...", $templatePath)); + $satis = json_decode( file_get_contents($templatePath), true) ; + + /* + * parameters + */ + $gitlabUrl = $input->getArgument('gitlab-url'); + if ( $gitlabUrl === static::DEFAULT_VALUE ) { + $gitlabUrlSet = isset($satis['config']['gitlab-domains']); + $gitlabUrlSet = $gitlabUrlSet && is_array($satis['config']['gitlab-domains']); + $gitlabUrlSet = $gitlabUrlSet && ! empty($satis['config']['gitlab-domains']); + if ($gitlabUrlSet) { + // if there is a gitlab domain already configured use it. + $gitlabUrl = 'https://' . reset($satis['config']['gitlab-domains']); + } else { + $gitlabUrl = static::DEFAULT_VALUE_GITLAB_URL; + } + } + $gitlabAuthToken = $input->getArgument('gitlab-token'); + $outputFile = $input->getOption('output'); + + /* + * Register gitlab domain to enable composer gitlab-* authentications + */ + $gitlabDomain = parse_url($gitlabUrl, PHP_URL_HOST); + if ( ! isset($satis['config']) ){ + $satis['config'] = array(); + } + if ( ! isset($satis['config']['gitlab-domains']) ){ + $satis['config']['gitlab-domains'] = array($gitlabDomain); + } else { + $satis['config']['gitlab-domains'][] = $gitlabDomain ; + } + + if ( ! $input->getOption('no-token') && ! empty($gitlabAuthToken) ){ + if ( ! isset($satis['config']['gitlab-token']) ){ + $satis['config']['gitlab-token'] = array(); + } + $satis['config']['gitlab-token'][$gitlabDomain] = $gitlabAuthToken; + } + + $this->processSatisConfiguration($input, $output, $satis); + + /* + * SCAN gitlab projects to find composer.json file in default branch + */ + $output->writeln(sprintf("Listing gitlab repositories from %s...", $gitlabUrl)); + $client = $this->createGitlabClient($gitlabUrl, $gitlabAuthToken); + for ($page = 1; $page <= self::MAX_PAGES; $page++) { + $projects = $client->projects()->all(array( + 'page' => $page, + 'per_page' => self::PER_PAGE + )); + if ( empty($projects) ){ + break; + } + foreach ($projects as $project) { + try { + $json = $client->repositoryFiles()->getRawFile($project['id'], 'composer.json', $project['default_branch']); + $composer = json_decode($json, true); + + $this->processProject($input, $output, $satis, $project, $composer); + } catch (\Exception $e) { + $this->displayProjectInfo( + $output, + $project, + 'composer.json not found', + OutputInterface::VERBOSITY_VERBOSE + ); + } + } + } + + $output->writeln("generate satis configuration file : $outputFile"); + $result = json_encode($satis, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + file_put_contents($outputFile, $result); + } + + /** + * display project information + */ + protected function displayProjectInfo( + OutputInterface $output, + array $project, + $message, + $verbosity = OutputInterface::VERBOSITY_NORMAL + ){ + $output->writeln(sprintf( + '%s (branch %s) : %s', + $project['name_with_namespace'], + $project['default_branch'], + $message + ),$verbosity); + } + + + /** + * Create gitlab client + * @param string $gitlabUrl + * @param string $gitlabAuthToken + * @return \Gitlab\Client + */ + protected function createGitlabClient($gitlabUrl, $gitlabAuthToken) { + /* + * create client with ssl verify disabled + */ + $guzzleClient = new \GuzzleHttp\Client(array( + 'verify' => false + )); + $httpClient = new \Http\Adapter\Guzzle6\Client($guzzleClient); + $httpClientBuilder = new \Gitlab\HttpClient\Builder($httpClient); + + $client = new \Gitlab\Client($httpClientBuilder); + $client->setUrl($gitlabUrl); + + // Authenticate to gitlab, if a token is provided + if ( ! empty($gitlabAuthToken) ) { + $client + ->authenticate($gitlabAuthToken, \Gitlab\Client::AUTH_URL_TOKEN) + ; + } + + return $client; + } + +} diff --git a/src/MBO/SatisGitlab/Command/GitlabToConfigCommand.php b/src/MBO/SatisGitlab/Command/GitlabToConfigCommand.php index ed9a788..18796b5 100644 --- a/src/MBO/SatisGitlab/Command/GitlabToConfigCommand.php +++ b/src/MBO/SatisGitlab/Command/GitlabToConfigCommand.php @@ -15,63 +15,54 @@ * * @author MBorne */ -class GitlabToConfigCommand extends Command { +class GitlabToConfigCommand extends GitlabCommandBase { - const PER_PAGE = 50; - const MAX_PAGES = 10000; - const HOMEPAGE_DEFAULT = '_default_'; - const HOMEPAGE_DEFAULT_VALUE = 'http://localhost/satis/'; + const DEFAULT_VALUE_HOMEPAGE = 'http://localhost/satis/'; protected function configure() { - $templatePath = realpath( dirname(__FILE__).'/../Resources/default-template.json' ); + parent::configure(); $this // the name of the command (the part after "bin/console") ->setName('gitlab-to-config') - // the short description shown while running "php bin/console list" - ->setDescription('generate satis configuration scanning gitlab repositories') - ->setHelp('look for composer.json in default gitlab branche, extract project name and register them in SATIS configuration') - ->addArgument('gitlab-url', InputArgument::REQUIRED) - ->addArgument('gitlab-token') - - // deep customization : template file extended with default configuration - ->addOption('template', null, InputOption::VALUE_REQUIRED, 'template satis.json extended with gitlab repositories', $templatePath) - // simple customization - ->addOption('homepage', null, InputOption::VALUE_REQUIRED, 'satis homepage', static::HOMEPAGE_DEFAULT) + ->addOption('homepage', null, InputOption::VALUE_REQUIRED, 'satis homepage', GitlabCommandBase::DEFAULT_VALUE) ->addOption('archive', null, InputOption::VALUE_NONE, 'enable archive mirroring') - - ->addOption('no-token', null, InputOption::VALUE_NONE, 'disable token writing in output configuration') - - // output configuration - ->addOption('output', 'O', InputOption::VALUE_REQUIRED, 'output config file', 'satis.json') ; } - protected function execute(InputInterface $input, OutputInterface $output) { - /* - * parameters - */ - $gitlabUrl = $input->getArgument('gitlab-url'); - $gitlabAuthToken = $input->getArgument('gitlab-token'); - $outputFile = $input->getOption('output'); - - /* - * load template satis.json file - */ - $templatePath = $input->getOption('template'); - $output->writeln(sprintf("Loading template %s...", $templatePath)); - $satis = json_decode( file_get_contents($templatePath), true) ; + protected function processProject(InputInterface $input, OutputInterface $output, array &$satis, array $project, array $composer) { + $project_name = $this->projectName($project, $composer); + $projectUrl = $project['http_url_to_repo']; + + $satis['repositories'][] = array( + 'type' => 'vcs', + 'url' => $projectUrl, + //TODO improve SSL management + 'options' => [ + "ssl" => [ + "verify_peer" => false, + "verify_peer_name" => false, + "allow_self_signed" => true + ] + ] + ); + $satis['require'][$project_name] = '*'; + $this->displayProjectInfo($output,$project, + "$project_name:*" + ); + } + protected function processSatisConfiguration(InputInterface $input, OutputInterface $output, array &$satis) { /* * customize according to command line options */ $homepage = $input->getOption('homepage'); - $homepage_default = $homepage === static::HOMEPAGE_DEFAULT; + $homepage_default = $homepage === GitlabCommandBase::DEFAULT_VALUE; $homepage_empty = !isset($satis['homepage']); if ( ! $homepage_default || $homepage_empty ) { - $satis['homepage'] = ($homepage_default) ? static::HOMEPAGE_DEFAULT_VALUE : $homepage; + $satis['homepage'] = ($homepage_default) ? static::DEFAULT_VALUE_HOMEPAGE : $homepage; } // mirroring @@ -83,127 +74,6 @@ protected function execute(InputInterface $input, OutputInterface $output) { 'skip-dev' => true ); } - - /* - * Register gitlab domain to enable composer gitlab-* authentications - */ - $gitlabDomain = parse_url($gitlabUrl, PHP_URL_HOST); - if ( ! isset($satis['config']) ){ - $satis['config'] = array(); - } - if ( ! isset($satis['config']['gitlab-domains']) ){ - $satis['config']['gitlab-domains'] = array($gitlabDomain); - }else{ - $satis['config']['gitlab-domains'][] = $gitlabDomain ; - } - - if ( ! $input->getOption('no-token') && ! empty($gitlabAuthToken) ){ - if ( ! isset($satis['config']['gitlab-token']) ){ - $satis['config']['gitlab-token'] = array(); - } - $satis['config']['gitlab-token'][$gitlabDomain] = $gitlabAuthToken; - } - - /* - * SCAN gitlab projects to find composer.json file in default branch - */ - $output->writeln(sprintf("Listing gitlab repositories from %s...", $gitlabUrl)); - $client = $this->createGitlabClient($gitlabUrl, $gitlabAuthToken); - for ($page = 1; $page <= self::MAX_PAGES; $page++) { - $projects = $client->projects()->all(array( - 'page' => $page, - 'per_page' => self::PER_PAGE - )); - if ( empty($projects) ){ - break; - } - foreach ($projects as $project) { - $projectUrl = $project['http_url_to_repo']; - try { - $json = $client->repositoryFiles()->getRawFile($project['id'], 'composer.json', $project['default_branch']); - $composer = json_decode($json, true); - - $projectName = isset($composer['name']) ? $composer['name'] : null; - if (is_null($projectName)) { - $this->displayProjectInfo($output,$project,'name not defined in composer.json'); - continue; - } - - $satis['repositories'][] = array( - 'type' => 'vcs', - 'url' => $projectUrl, - //TODO improve SSL management - 'options' => [ - "ssl" => [ - "verify_peer" => false, - "verify_peer_name" => false, - "allow_self_signed" => true - ] - ] - ); - $satis['require'][$projectName] = '*'; - $this->displayProjectInfo($output,$project, - "$projectName:*" - ); - } catch (\Exception $e) { - $this->displayProjectInfo($output,$project, - 'composer.json not found', - OutputInterface::VERBOSITY_VERBOSE - ); - } - } - } - - $output->writeln("generate satis configuration file : $outputFile"); - $result = json_encode($satis, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); - file_put_contents($outputFile, $result); - } - - /** - * display project information - */ - protected function displayProjectInfo( - OutputInterface $output, - array $project, - $message, - $verbosity = OutputInterface::VERBOSITY_NORMAL - ){ - $output->writeln(sprintf( - '%s (branch %s) : %s', - $project['name_with_namespace'], - $project['default_branch'], - $message - ),$verbosity); - } - - - /** - * Create gitlab client - * @param string $gitlabUrl - * @param string $gitlabAuthToken - * @return \Gitlab\Client - */ - protected function createGitlabClient($gitlabUrl, $gitlabAuthToken) { - /* - * create client with ssl verify disabled - */ - $guzzleClient = new \GuzzleHttp\Client(array( - 'verify' => false - )); - $httpClient = new \Http\Adapter\Guzzle6\Client($guzzleClient); - $httpClientBuilder = new \Gitlab\HttpClient\Builder($httpClient); - - $client = new \Gitlab\Client($httpClientBuilder); - $client->setUrl($gitlabUrl); - - // Authenticate to gitlab, if a token is provided - if ( ! empty($gitlabAuthToken) ) { - $client - ->authenticate($gitlabAuthToken, \Gitlab\Client::AUTH_URL_TOKEN) - ; - } - - return $client; } } From 9748a57f5e06aefc0f999dfc3214285f8f03c140 Mon Sep 17 00:00:00 2001 From: Rich Gerdes Date: Mon, 18 Jun 2018 16:49:58 -0400 Subject: [PATCH 2/4] Add new commend to fetch gitlab package dependencies to create a cache of required packages. --- bin/satis-gitlab | 1 + .../GitlabDependenciesToConfigCommand.php | 58 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 src/MBO/SatisGitlab/Command/GitlabDependenciesToConfigCommand.php diff --git a/bin/satis-gitlab b/bin/satis-gitlab index ad91a43..39983d0 100755 --- a/bin/satis-gitlab +++ b/bin/satis-gitlab @@ -27,4 +27,5 @@ if ((!$loader = includeIfExists(__DIR__.'/../vendor/autoload.php')) && (!$loader */ $application = new Composer\Satis\Console\Application(); $application->add(new \MBO\SatisGitlab\Command\GitlabToConfigCommand()); +$application->add(new \MBO\SatisGitlab\Command\GitlabDependenciesToConfigCommand()); $application->run(); diff --git a/src/MBO/SatisGitlab/Command/GitlabDependenciesToConfigCommand.php b/src/MBO/SatisGitlab/Command/GitlabDependenciesToConfigCommand.php new file mode 100644 index 0000000..2a61fc3 --- /dev/null +++ b/src/MBO/SatisGitlab/Command/GitlabDependenciesToConfigCommand.php @@ -0,0 +1,58 @@ +setName('gitlab-dependencies-to-config') + ; + } + + protected function processProject(InputInterface $input, OutputInterface $output, array &$satis, array $project, array $composer) { + $project_name = $this->projectName($project, $composer); + + $local_packages = array(); + if (isset($composer['repositories']) && is_array($composer['repositories'])) { + foreach ($composer['repositories'] as $repository) { + // find list of packages that are defined locally. + if ($repository['type'] === 'package' && isset ($repository['package'])) { + $local_packages[] = $repository['package']['name']; + } + } + } + + if (isset($composer['require']) && is_array($composer['require'])) { + foreach (array_keys($composer['require']) as $dep_name) { + if (in_array($dep_name, $local_packages)) { + $output->writeln(sprintf(" Skipping local package %s...", $dep_name)); + continue; + } else if (in_array($dep_name, array_keys($satis['require']))) { + continue; + } + $satis['require'][$dep_name] = '*'; + + } + } + + } + +} From 93a95ef2f4342a61e8123ba93b4343478e1bc337 Mon Sep 17 00:00:00 2001 From: Rich Gerdes Date: Mon, 18 Jun 2018 17:00:47 -0400 Subject: [PATCH 3/4] Add gitlab-dependencies-to-config command to readme. --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index cd10316..3b4000e 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,19 @@ Some command line options provide a basic customization options. You may also us [default-template.json](src/MBO/SatisGitlab/Resources/default-template.json) +### Additional Commands + +This project also provides another command for fetching the dependencies used by projects found in the gitlab instance. + +To use this command, run: + +```bash +bin/satis-gitlab gitlab-dependencies-to-config \ + --output satis.json \ + https://gitlab.example.org [GitlabToken] +``` + +You can chain this command, and append to the output of the `gitlab-to-config` command, buy supplying `--template` with the output file. ## Requirements From 358819ce21761bcdce46279d74fbf104f6ca219a Mon Sep 17 00:00:00 2001 From: Rich Gerdes Date: Tue, 19 Jun 2018 14:56:33 -0400 Subject: [PATCH 4/4] Skip duplicate gitlab instances. --- src/MBO/SatisGitlab/Command/GitlabCommandBase.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MBO/SatisGitlab/Command/GitlabCommandBase.php b/src/MBO/SatisGitlab/Command/GitlabCommandBase.php index 24b5635..39a5d2e 100644 --- a/src/MBO/SatisGitlab/Command/GitlabCommandBase.php +++ b/src/MBO/SatisGitlab/Command/GitlabCommandBase.php @@ -100,7 +100,7 @@ protected function execute(InputInterface $input, OutputInterface $output) { } if ( ! isset($satis['config']['gitlab-domains']) ){ $satis['config']['gitlab-domains'] = array($gitlabDomain); - } else { + } else if ( ! in_array($gitlabDomain, $satis['config']['gitlab-domains']) ) { $satis['config']['gitlab-domains'][] = $gitlabDomain ; }