Merge branch 'ISAICP-8996' into 'develop'

ISAICP-8996: Onboard Dutch temporary catalogue.

See merge request digit/digit-joinup-dev!1563

web/modules/custom/eu_oss_catalogue/README.md

@@ -245,6 +245,11 @@ Run the following, to fetch from
 ./vendor/bin/drush eu-oss:fetch hosting_platform:code_europa_eu
 ```
 
+Run the following, to fetch from
+Fetch solutions from Dutch catalogue
+```bash
+./vendor/bin/drush eu-oss:fetch dutch_catalogue
+
 This command queries the Developers Italia provider endpoint, fetches the data
 and populates the queue. Later, the cronjob is triggering the queue worker which
 is gradually creating or updating the OSS Solutions.

web/modules/custom/eu_oss_catalogue/src/Plugin/eu_oss_catalogue/Provider/DutchCatalogue.php

+<?php
+
+declare(strict_types=1);
+
+namespace Drupal\eu_oss_catalogue\Plugin\eu_oss_catalogue\Provider;
+
+use Drupal\Component\Serialization\Json;
+use Drupal\Core\Logger\LoggerChannelInterface;
+use Drupal\eu_oss_catalogue\ApiGuesserInterface;
+use Drupal\eu_oss_catalogue\EuOssCataloguePluginManagerInterface;
+use Drupal\eu_oss_catalogue\EuOssCatalogueProviderPluginBase;
+use Drupal\eu_oss_catalogue\Exception\HostingPlatformFetchException;
+use Drupal\eu_oss_catalogue\Exception\ProviderFetchException;
+use Drupal\eu_oss_catalogue\Model\Project;
+use GuzzleHttp\Client;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+/**
+ * Dutch catalogue provider.
+ *
+ * @EuOssCatalogueProvider(
+ *   id = "dutch_catalogue",
+ *   label = @Translation("Dutch catalogue"),
+ *   description = @Translation("Dutch open source software solutions for the public administrations."),
+ *   parser = "publiccode_yml",
+ * )
+ */
+class DutchCatalogue extends EuOssCatalogueProviderPluginBase {
+
+  /**
+   * Dutch catalogue endpoint.
+   */
+  protected const string ENDPOINT = 'https://gist.githubusercontent.com/dvh/95a09cf9df9b4567c256c188a0e8f4b2/raw';
+
+  /**
+   * List of projects fetched from Dutch catalogue API.
+   *
+   * @var array[][]
+   *   A list of fetched projects keyed by the coding platform base URL. The
+   *   values are associative arrays keyed by namespace and having a list of
+   *   project IDs as values.
+   */
+  protected array $fetched;
+
+  /**
+   * Fetched URLs collector.
+   */
+  protected array $fetchedUrls = [];
+
+  /**
+   * Processed URLs collector.
+   */
+  protected array $processedUrls = [];
+
+  /**
+   * Number of solutions provided by API.
+   *
+   * @var int
+   */
+  protected int $numberOfProvidedSolutions = 0;
+
+  public function __construct(
+    array $configuration,
+    $pluginId,
+    $pluginDefinition,
+    protected EuOssCataloguePluginManagerInterface $parserManager,
+    protected LoggerChannelInterface $logger,
+    protected Client $httpClient,
+    protected EuOssCataloguePluginManagerInterface $platformManager,
+    protected ApiGuesserInterface $apiGuesser,
+  ) {
+    parent::__construct($configuration, $pluginId, $pluginDefinition, $parserManager, $logger);
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public static function create(ContainerInterface $container, array $configuration, $pluginId, $pluginDefinition): self {
+    return new static(
+      $configuration,
+      $pluginId,
+      $pluginDefinition,
+      $container->get('plugin.manager.eu_oss_catalogue.parser'),
+      $container->get('logger.channel.eu_oss_catalogue'),
+      $container->get('http_client'),
+      $container->get('plugin.manager.eu_oss_catalogue.hosting_platform'),
+      $container->get('eu_oss_catalogue.git_remote.api_guesser'),
+    );
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function fetch(): iterable {
+    if (!isset($this->fetched)) {
+      $this->getData();
+    }
+
+    $collectedExceptions = [];
+    foreach ($this->fetched as $baseUrl => $namespaces) {
+      // Remove this entry to avoid fetching again on the next iteration.
+      unset($this->fetched[$baseUrl]);
+
+      $platformConfig = [
+        'baseUrl' => $baseUrl,
+        'projects' => $namespaces['projects'],
+      ];
+      /** @var \Drupal\eu_oss_catalogue\EuOssCatalogueHostingPlatformInterface $hostingPlatform */
+      $hostingPlatform = $this->platformManager->createInstance($namespaces['api']->pluginId, $platformConfig);
+
+      /** @var \Drupal\eu_oss_catalogue\Model\Project $project */
+      try {
+        foreach ($hostingPlatform->getProjects([
+          'publiccodeYml' => 'publiccode.yml',
+          'publiccodeYaml' => 'publiccode.yaml',
+        ]) as $project) {
+          $this->processedUrls[] = $project->url;
+
+          // Try to use "publiccode.yaml".
+          if (isset($project->files['publiccodeYaml'])) {
+            $values = $project->toArray();
+            $values['files']['publiccodeYml'] = $values['files']['publiccodeYaml'];
+            unset($values['files']['publiccodeYaml']);
+            yield new Project(...$values);
+            continue;
+          }
+
+          yield $project;
+        }
+      }
+      catch (HostingPlatformFetchException $exception) {
+        // Collect provider fetch exceptions, and throw them at the end of the
+        // fetch operation.
+        $collectedExceptions[] = $exception->getMessage();
+      }
+
+      // Only log when all solutions were processed.
+      if (empty($this->fetched)) {
+        $this->logResults();
+      }
+    }
+
+    if (!empty($collectedExceptions)) {
+      throw new ProviderFetchException(implode(PHP_EOL, $collectedExceptions));
+    }
+  }
+
+  /**
+   * Gets data.
+   *
+   * @throws \Drupal\eu_oss_catalogue\Exception\ProviderFetchException
+   */
+  protected function getData(): void {
+    $this->fetched = [];
+    try {
+      $response = $this->httpClient->get(static::ENDPOINT);
+
+      // Fix not valid data.
+      $responseData = str_replace('""publiccode_url":', '", "publiccode_url":', (string) $response->getBody());
+      $responseData = Json::decode($responseData);
+    }
+    catch (\Exception $exception) {
+      throw new ProviderFetchException(
+        $exception->getMessage()
+      );
+    }
+
+    $this->numberOfProvidedSolutions = count($responseData);
+    foreach ($responseData as $row) {
+      $required = ['url', 'owner_name', 'name'];
+      foreach ($required as $field) {
+        if (empty($row[$field])) {
+          $this->logger->error('Lacks the mandatory @field in @row', [
+            '@field' => $row[$field],
+            '@row' => print_r($row, TRUE),
+          ]);
+          continue 2;
+        }
+      }
+
+      if (!$api = $this->apiGuesser->getHostingPlatformApi($row['url'])) {
+        $this->logger->error('Can\'t determinate hosting platform for @repository repository',
+          ['@repository' => $row['url']]);
+        continue;
+      }
+
+      $this->fetched[$api->baseUrl]['api'] = $api;
+      $this->fetched[$api->baseUrl]['projects'][$row['owner_name']][] = $row['name'];
+      $this->fetchedUrls[] = $row['url'];
+    }
+  }
+
+  /**
+   * Logs statistics about fetched and processed OSS Solutions.
+   */
+  protected function logResults(): void {
+    if ($unProcessed = array_diff($this->fetchedUrls, $this->processedUrls)) {
+      $this->logger->warning('Processed @processed out of @fetched fetched solutions from Dutch catalogue. The following @unprocessed solutions failed: ' . implode(', ', $unProcessed), [
+        '@processed' => count($this->processedUrls),
+        '@fetched' => $this->numberOfProvidedSolutions,
+        '@unprocessed' => count($unProcessed),
+        'list' => $unProcessed,
+      ]);
+    }
+    else {
+      $this->logger->info('Processed @processed out of @fetched fetched solutions from Dutch catalogue.', [
+        '@processed' => count($this->processedUrls),
+        '@fetched' => $this->numberOfProvidedSolutions,
+      ]);
+    }
+  }
+
+}

web/modules/custom/eu_oss_catalogue/tests/fixtures/expectancy/DutchCatalogueFetchTest.testImport.yml

+GitHub solution 1:
+  en:
+    langcode: en
+    default_langcode: '1'
+    status: '1'
+    oss_available_languages:
+      - it
+      - en
+      - de
+    oss_categories:
+      - digital-citizenship
+    oss_development_status: stable
+    oss_git_url: https://github.com/dummy-namespace-1/dummy-name-1
+    oss_license: AGPL-3.0-or-later
+    oss_logo: https://raw.githubusercontent.com/dummy-namespace-1/dummy-name-1/HEAD/images/logo-solution-1.png
+    oss_git_forks: 103
+    oss_git_stars: 123
+    oss_git_open_issues: 143
+    oss_maintenance_contacts:
+      - Peter Gregory
+    oss_maintenance_contractors:
+      - ACME Ltd
+      - Das Auto GmbH
+    oss_maintenance_type: contract
+    oss_owner: Sin City
+    oss_platforms:
+      - web
+    oss_publiccode_yml_version: '0.2'
+    oss_release_date: '2017-05-01'
+    oss_software_type: standalone/web
+    oss_software_version: '1.4'
+    oss_source:
+      - dutch_catalogue
+    oss_source_hash: _RnRGlJcP5QA_tiadP_JTMJyyOypigHzRYjIY-LlQMM
+    oss_long_description: |
+      [EN] Solution 1: Long description
+      **Some bold text**
+      * Some list item
+      * Another list item
+
+      _Some italic text_
+    oss_short_description: '[EN] Solution 1: **Short description**'
+    oss_features:
+      - '[EN] Solution 1: Feature 1'
+      - '[EN] Solution 1: Feature 2'
+      - '[EN] Solution 1: Feature 3'
+    oss_screenshots:
+      - https://raw.githubusercontent.com/dummy-namespace-1/dummy-name-1/HEAD/Solution_1_EN_Screenshot1.jpg
+      - https://raw.githubusercontent.com/dummy-namespace-1/dummy-name-1/HEAD/Solution_1_EN_Screenshot2.jpg
+    oss_git_open_pull_requests: 3
+  de:
+    langcode: de
+    default_langcode: '0'
+    oss_features:
+      - '[DE] Solution 1: Feature 1'
+    oss_long_description: |
+      [DE] Solution 1: Long description
+      **Some bold text**
+      * Some list item
+      * Another list item
+
+      _Some italic text_
+    oss_short_description: '[DE] Solution 1: **Short description**'
+    oss_screenshots:
+      - https://raw.githubusercontent.com/dummy-namespace-1/dummy-name-1/HEAD/Solution_1_DE_Screenshot1.jpg
+      - https://raw.githubusercontent.com/dummy-namespace-1/dummy-name-1/HEAD/Solution_1_DE_Screenshot3.jpg
+      - https://raw.githubusercontent.com/dummy-namespace-1/dummy-name-1/HEAD/Solution_1_DE_Screenshot4.jpg
+  it:
+    langcode: it
+    default_langcode: '0'
+    oss_features:
+      - '[IT] Solution 1: Feature 1'
+      - '[IT] Solution 1: Feature 2'
+    oss_long_description: |
+      [IT] Solution 1: Long description
+      **Some bold text**
+      * Some list item
+      * Another list item
+
+      _Some italic text_
+    oss_short_description: '[IT] Solution 1: **Short description**'
+    oss_screenshots:
+      - https://raw.githubusercontent.com/dummy-namespace-1/dummy-name-1/HEAD/Solution_1_IT_Screenshot1.jpg
+
+Project 1.1:
+  it:
+    langcode: it
+    default_langcode: '0'
+    oss_categories:
+      - data-collection
+    oss_git_url: https://limited.self-hosted.gitlab.example.com/group1/project-1-1
+    oss_git_description: 'Group 1: Project 1: description'
+    oss_default_branch: branch-project-1-1
+    oss_software_version: '2.0.0-alpha12'
+    oss_release_date: '2010-07-08'
+    oss_platform_reference: gid://gitlab/Project/11
+    oss_logo: https://limited.self-hosted.gitlab.example.com/group1/project-1-1/raw/HEAD/images/logo-of-project-1-1.svg
+    oss_git_forks: 206
+    oss_git_stars: 226
+    oss_git_open_issues: 246
+    oss_publiccode_yml_version: '0.2'
+    oss_documentation: https://example.com/group1/project-1-1/it/docs.md
+    oss_features:
+      - '[IT] feat 1'
+      - '[IT] feat 2'
+      - '[IT] feat 3'
+    oss_long_description: '[IT] Group1: Project 1: Long description'
+    oss_short_description: '[IT] Group1: Project 1: Short description'
+    oss_screenshots:
+      - 'https://limited.self-hosted.gitlab.example.com/group1/project-1-1/raw/HEAD/Group_1_Project1_IT_Screenshot.jpg'
+    oss_git_open_pull_requests: 32
+  de:
+    langcode: de
+    oss_documentation: https://example.com/group1/project-1-1/de/docs.md
+    oss_features:
+      - '[DE] feat 1'
+      - '[DE] feat 2'
+    oss_long_description: '[DE] Group1: Project 1: Long description'
+    oss_short_description: '[DE] Group1: Project 1: Short description'
+
+Project 1.2:
+  de:
+    langcode: de
+    default_langcode: '0'
+    oss_categories:
+      - data-collection
+      - data-analytics
+    oss_git_url: https://limited.self-hosted.gitlab.example.com/group1/project-1-2
+    oss_git_description: 'Group 1: Project 2: description'
+    oss_default_branch: branch-project-1-2
+    oss_software_version: '220.0.1'
+    oss_release_date: '2013-08-19'
+    oss_platform_reference: gid://gitlab/Project/12
+    oss_logo: https://all.self-hosted.gitlab.example.com/uploads/-/system/project/avatar/12/logo.jpg
+    oss_git_forks: 207
+    oss_git_stars: 227
+    oss_git_open_issues: 247
+    oss_publiccode_yml_version: '0.2'
+    oss_documentation: https://example.com/group1/project-1-2/de/docs.md
+    oss_features:
+      - '[DE] feat 1'
+      - '[DE] feat 2'
+    oss_long_description: '[DE] Group1: Project 2: Long description'
+    oss_short_description: '[DE] Group1: Project 2: Short description'
+    oss_git_open_pull_requests: 2
+  it:
+    langcode: it
+    default_langcode: '0'
+    oss_documentation: https://example.com/group1/project-1-2/it/docs.md
+    oss_features:
+      - '[IT] feat 1'
+      - '[IT] feat 2'
+    oss_long_description: '[IT] Group1: Project 2: Long description'
+    oss_short_description: '[IT] Group1: Project 2: Short description'
+  en:
+    langcode: en
+    default_langcode: '1'
+    oss_documentation: https://example.com/group1/project-1-2/en/docs.md
+    oss_features:
+      - '[EN] feat 1'
+    oss_long_description: '[EN] Group1: Project 2: Long description'
+    oss_short_description: '[EN] Group1: Project 2: Short description'
+    oss_used_by:
+      - 'Abbanoa'
+      - 'abbanoa'
+      - 'abbanoa '
+      - 'Comune Di Malè'
+      - 'Comune Di Male'
+      - 'comune di male'

web/modules/custom/eu_oss_catalogue/tests/fixtures/http_response/dutch_catalogue.yml

+- id: 3248
+  source: github
+  owner_name: dummy-namespace-1
+  name: dummy-name-1
+  url: 'https://github.com/dummy-namespace-1/dummy-name-1.git'
+  description: 'Frontend of the SIA (Signalen Informatievoorziening Amsterdam)'
+  last_change: '2024-07-08T14:56:18.000Z'
+  stars: 8
+  avatar_url: 'https://avatars.githubusercontent.com/u/14022058?v=4'
+  fork_count: 24
+  issue_open_count: 0
+  last_fetched_at: '2024-07-11T13:01:28.157Z'
+  merge_request_open_count: 15
+  archived: false
+  organization_id: 15
+  readme_url: ''
+  slug: 'amsterdam-signals-frontend-25698'
+  publiccode_url: 'https://github.com/Amsterdam/signals-frontend/blob/main/publiccode.yml'
+
+- id: 6011
+  source: 'gitlab'
+  owner_name: 'group1'
+  name: 'project-1-1'
+  url: 'https://limited.self-hosted.gitlab.example.com/group1/project-1-1'
+  description: 'Implementatie mbv IaC van de verwerkinglogging standaard'
+  last_change: '2024-07-11T00:05:41.000Z'
+  stars: 5
+  avatar_url: 'https://avatars.githubusercontent.com/u/29945117?v=4'
+  fork_count: 0
+  issue_open_count: 14
+  last_fetched_at: '2024-07-11T13:22:46.916Z'
+  merge_request_open_count: 0
+  archived: false
+  organization_id: 9
+  readme_url: ''
+  slug: 'gemeentenijmegen-verwerkingenlogging-33391'
+  publiccode_url: 'https://github.com/GemeenteNijmegen/verwerkingenlogging/blob/main/publiccode.yaml'
+
+- id: 5967
+  source: 'gitlab'
+  owner_name: 'group1'
+  name: 'project-1-2'
+  url: 'https://limited.self-hosted.gitlab.example.com/group1/project-1-2'
+  description: 'This project is a proof of concept of a service that can translate StUF ZDS 1.2 SOAP messages to their corresponding ZGW 1.0 API calls.'
+  last_change: '2024-02-13T10:41:21.000Z'
+  stars: 0
+  avatar_url: 'https://avatars.githubusercontent.com/u/15446184?v=4'
+  fork_count: 8
+  issue_open_count: 1
+  last_fetched_at: '2024-07-11T13:22:04.801Z'
+  merge_request_open_count: 4
+  archived: false
+  organization_id: 8
+  readme_url: ''
+  slug: 'haarlem-zds-stuf-to-zgw-api-translator-38688'
+  publiccode_url: 'https://github.com/Haarlem/zds-stuf-to-zgw-api-translator/blob/master/publiccode.yml'

web/modules/custom/eu_oss_catalogue/tests/modules/eu_oss_catalogue_test/src/Plugin/ServiceMock/DutchCatalogue.php

+<?php
+
+declare(strict_types=1);
+
+namespace Drupal\eu_oss_catalogue_test\Plugin\ServiceMock;
+
+use Drupal\Component\Serialization\Json;
+use Drupal\Core\Plugin\PluginBase;
+use Drupal\Core\Serialization\Yaml;
+use Drupal\eu_oss_catalogue_test\IntegrationTestTrait;
+use Drupal\http_request_mock\ServiceMockPluginInterface;
+use GuzzleHttp\Psr7\Response;
+use Psr\Http\Message\RequestInterface;
+use Psr\Http\Message\ResponseInterface;
+
+/**
+ * Intercepts any HTTP requests made to Dutch catalogue API.
+ *
+ * @ServiceMock(
+ *   id = "dutch_catalogue",
+ *   label = @Translation("Dutch catalogue API"),
+ *   weight = 0,
+ * )
+ */
+class DutchCatalogue extends PluginBase implements ServiceMockPluginInterface {
+
+  use IntegrationTestTrait;
+
+  /**
+   * {@inheritdoc}
+   */
+  public function applies(RequestInterface $request, array $options): bool {
+    return !$this->isIntegrationTest() && ($request->getUri()
+      ->getHost() === 'gist.githubusercontent.com');
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getResponse(RequestInterface $request, array $options): ResponseInterface {
+    $response = Json::encode(Yaml::decode(file_get_contents(__DIR__ . '/../../../../../fixtures/http_response/dutch_catalogue.yml')));
+    return new Response(200, [
+      'Content-Type' => 'application/json',
+      'Content-Length' => strlen($response),
+    ], $response);
+  }
+
+}

web/modules/custom/eu_oss_catalogue/tests/src/Functional/DutchCatalogueFetchTest.php

+<?php
+
+declare(strict_types=1);
+
+namespace Drupal\Tests\eu_oss_catalogue\Functional;
+
+/**
+ * EU OSS Catalogue test.
+ *
+ * @group eu_oss_catalogue
+ */
+class DutchCatalogueFetchTest extends FetchTestBase {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected array $controlledVocabTerms = [
+    'oss_licence' => ['AGPL-3.0-or-later'],
+    'oss_category' => ['digital-citizenship'],
+  ];
+
+  /**
+   * @covers
+   */
+  public function testImport(): void {
+    $this->fetchSolutions('dutch_catalogue', 3);
+
+    $this->assertOssSolution('GitHub solution 1', 'en');
+    $this->assertOssSolution('Project 1.1', 'it');
+    $this->assertOssSolution('Project 1.2', 'de');
+  }
+
+}