From 2a7c71fcc7dc14d348bcc6cdac8abcfd69ebf32d Mon Sep 17 00:00:00 2001 From: "Dusan Mijatovic (PC2026)" Date: Fri, 30 Oct 2026 18:01:38 +0100 Subject: [PATCH 1/4] chore: add source code of rsd services v3.4.1 for critical security updates --- authentication/.editorconfig | 17 + authentication/Dockerfile | 21 + authentication/README.md | 47 + authentication/pom.xml | 215 ++++ .../rsd/authentication/Account.java | 13 + .../rsd/authentication/AccountInfo.java | 20 + .../rsd/authentication/AzureLogin.java | 69 ++ .../rsd/authentication/Config.java | 184 ++++ .../rsd/authentication/HelmholtzIdLogin.java | 228 +++++ .../rsd/authentication/JwtCreator.java | 69 ++ .../rsd/authentication/JwtVerifier.java | 32 + .../rsd/authentication/LinkedinLogin.java | 69 ++ .../rsd/authentication/Login.java | 13 + .../rsd/authentication/Main.java | 389 ++++++++ .../rsd/authentication/OpenIdInfo.java | 20 + .../rsd/authentication/OpenidProvider.java | 26 + .../OpenidProviderAccessMethod.java | 35 + .../rsd/authentication/OrcidLogin.java | 71 ++ .../rsd/authentication/PostgrestAccount.java | 319 ++++++ .../authentication/QueryParameterBuilder.java | 39 + .../RsdAccountInviteException.java | 16 + .../RsdAuthenticationException.java | 13 + .../rsd/authentication/RsdProviderData.java | 18 + .../rsd/authentication/RsdProviders.java | 347 +++++++ .../authentication/RsdResponseException.java | 26 + .../rsd/authentication/SurfconextLogin.java | 69 ++ .../rsd/authentication/Utils.java | 107 ++ authentication/src/main/resources/logback.xml | 25 + .../authentication/HelmholtzIdLoginTest.java | 141 +++ .../rsd/authentication/MainTest.java | 39 + .../authentication/PostgrestAccountTest.java | 104 ++ .../QueryParameterBuilderTest.java | 48 + .../rsd/authentication/RsdProvidersTest.java | 125 +++ .../rsd/authentication/UtilsTest.java | 119 +++ backend-postgrest/Dockerfile | 7 + background-services/Dockerfile | 17 + background-services/README.md | 20 + background-services/pom.xml | 83 ++ .../nl/esciencecenter/rsd/MainDatabase.java | 57 ++ .../src/main/resources/logback.xml | 25 + database/.editorconfig | 17 + database/000-fill-authenticator-password.sh | 7 + database/001-setup-basic-roles.sql | 28 + database/002-create-account-table.sql | 143 +++ database/003-create-image-table.sql | 88 ++ database/004-create-software-table.sql | 68 ++ .../005-create-relations-for-software.sql | 270 +++++ database/006-create-project-table.sql | 85 ++ .../007-create-relations-for-projects.sql | 91 ++ database/008-create-community-table.sql | 351 +++++++ database/010-create-research-domain.sql | 515 ++++++++++ database/011-create-mention-table.sql | 169 ++++ database/012-create-release-table.sql | 55 + database/013-create-organisation-table.sql | 195 ++++ database/014-create-keyword-and-category.sql | 275 +++++ database/015-inter-relation-tables.sql | 174 ++++ database/016-create-maintainer-tables.sql | 277 ++++++ database/017-meta-pages.sql | 56 ++ database/018-create-backend-logs-table.sql | 40 + database/020-row-level-security.sql | 816 +++++++++++++++ database/021-create-global-triggers.sql | 116 +++ database/022-create-maintenance-tables.sql | 47 + database/023-news.sql | 112 +++ database/024-remote-rsd.sql | 89 ++ database/025-rsd-info.sql | 53 + database/026-user-profile.sql | 143 +++ database/100-create-api-views.sql | 926 +++++++++++++++++ database/102-create-account-functions.sql | 45 + database/104-person-views.sql | 186 ++++ database/104-software-views.sql | 833 ++++++++++++++++ database/106-project-views.sql | 940 ++++++++++++++++++ database/108-community-views.sql | 613 ++++++++++++ database/109-category-functions.sql | 83 ++ database/113-organisation-views.sql | 728 ++++++++++++++ database/114-keyword-category-views.sql | 74 ++ database/118-backend-logs-views.sql | 60 ++ database/124-aggregated-software-views.sql | 315 ++++++ database/125-global-search.sql | 187 ++++ database/126-public-persons-overview.sql | 91 ++ database/201-nassa-import.sql | 239 +++++ database/999-add-categories.sql.example | 102 ++ database/Dockerfile | 9 + database/sonar-project.properties | 18 + docker-compose.yml | 51 +- scrapers/.editorconfig | 20 + scrapers/Dockerfile | 22 + scrapers/README.md | 36 + scrapers/jobs.cron | 15 + scrapers/pom.xml | 171 ++++ .../rsd/scraper/BibtexParser.java | 335 +++++++ .../nl/esciencecenter/rsd/scraper/Config.java | 204 ++++ .../rsd/scraper/InvalidInputException.java | 13 + .../rsd/scraper/RsdRateLimitException.java | 24 + .../rsd/scraper/RsdResponseException.java | 26 + .../nl/esciencecenter/rsd/scraper/Utils.java | 371 +++++++ .../scraper/aggregator/MainAggregator.java | 161 +++ .../aggregator/PostgrestConnector.java | 106 ++ .../aggregator/RemoteRsdConnector.java | 51 + .../rsd/scraper/aggregator/RemoteRsdData.java | 21 + .../rsd/scraper/doi/CitationData.java | 27 + .../rsd/scraper/doi/CrossrefMention.java | 136 +++ .../doi/DataCiteReleaseRepository.java | 95 ++ .../doi/DataciteMentionRepository.java | 229 +++++ .../esciencecenter/rsd/scraper/doi/Doi.java | 63 ++ .../scraper/doi/ExternalMentionRecord.java | 26 + .../rsd/scraper/doi/MainCitations.java | 93 ++ .../rsd/scraper/doi/MainMentions.java | 211 ++++ .../rsd/scraper/doi/MainReleases.java | 65 ++ .../rsd/scraper/doi/MentionType.java | 28 + .../rsd/scraper/doi/OpenAlexConnector.java | 297 ++++++ .../rsd/scraper/doi/OpenalexId.java | 76 ++ .../doi/PostgrestCitationRepository.java | 100 ++ .../doi/PostgrestMentionRepository.java | 163 +++ .../doi/PostgrestReleaseRepository.java | 99 ++ .../rsd/scraper/doi/ReleaseData.java | 27 + .../rsd/scraper/doi/RsdMentionIds.java | 15 + .../rsd/scraper/doi/RsdMentionRecord.java | 16 + .../rsd/scraper/git/BasicGitData.java | 15 + .../rsd/scraper/git/BasicGitDatabaseData.java | 15 + .../rsd/scraper/git/BasicRepositoryData.java | 11 + .../git/BasicRepositoryDataWithHistory.java | 12 + .../rsd/scraper/git/CodePlatformProvider.java | 10 + .../rsd/scraper/git/CommitData.java | 15 + .../rsd/scraper/git/CommitsPerWeek.java | 80 ++ .../scraper/git/ContributorDatabaseData.java | 15 + .../rsd/scraper/git/FourTuGitScraper.java | 69 ++ .../rsd/scraper/git/GitScraper.java | 21 + .../rsd/scraper/git/GithubScraper.java | 240 +++++ .../rsd/scraper/git/GitlabScraper.java | 192 ++++ .../rsd/scraper/git/LanguagesData.java | 15 + .../rsd/scraper/git/MainBasicData.java | 110 ++ .../rsd/scraper/git/MainCommits.java | 152 +++ .../rsd/scraper/git/MainContributors.java | 157 +++ .../scraper/git/MainProgrammingLanguages.java | 154 +++ .../rsd/scraper/git/PostgrestConnector.java | 167 ++++ .../license/GitHubSpdxLicenseRepository.java | 53 + .../rsd/scraper/license/SpdxLicense.java | 13 + .../rsd/scraper/nassa/MainNassa.java | 166 ++++ .../rsd/scraper/nassa/NassaReadmeParser.java | 43 + .../rsd/scraper/nassa/NassaSoftwareEntry.java | 369 +++++++ .../scraper/nassa/RsdNassaParseException.java | 13 + .../BasicPackageManagerData.java | 15 + .../package_manager/MainPackageManager.java | 154 +++ .../package_manager/PackageManagerType.java | 23 + .../package_manager/PostgrestConnector.java | 70 ++ .../scrapers/AnacondaScraper.java | 44 + .../package_manager/scrapers/CranScraper.java | 53 + .../scrapers/CratesScraper.java | 46 + .../scrapers/DockerHubScraper.java | 75 ++ .../scrapers/FourTuScraper.java | 22 + .../package_manager/scrapers/GoScraper.java | 48 + .../scrapers/MavenScraper.java | 57 ++ .../package_manager/scrapers/NpmScraper.java | 45 + .../scrapers/PackageManagerScraper.java | 49 + .../package_manager/scrapers/PixiScraper.java | 71 ++ .../package_manager/scrapers/PypiScraper.java | 44 + .../rsd/scraper/ror/MainRor.java | 68 ++ .../scraper/ror/OrganisationDatabaseData.java | 19 + .../rsd/scraper/ror/RorData.java | 19 + .../esciencecenter/rsd/scraper/ror/RorId.java | 100 ++ .../scraper/ror/RorPostgrestConnector.java | 82 ++ .../rsd/scraper/ror/RorScraper.java | 133 +++ scrapers/src/main/resources/logback.xml | 25 + scrapers/src/main/resources/oaipmh.py | 142 +++ scrapers/src/main/resources/releases.py | 310 ++++++ scrapers/src/main/resources/requirements.txt | 11 + scrapers/src/main/resources/zotero.py | 187 ++++ .../rsd/scraper/BibtexParserTest.java | 87 ++ .../esciencecenter/rsd/scraper/UtilsIT.java | 43 + .../esciencecenter/rsd/scraper/UtilsTest.java | 32 + .../doi/DataciteMentionRepositoryTest.java | 28 + .../rsd/scraper/doi/DoiTest.java | 54 + .../rsd/scraper/doi/MainMentionsTest.java | 57 ++ .../scraper/doi/OpenAlexConnectorTest.java | 43 + .../rsd/scraper/doi/OpenalexIdTest.java | 49 + .../rsd/scraper/git/CommitsPerWeekTest.java | 118 +++ .../rsd/scraper/git/GithubScraperIT.java | 61 ++ .../rsd/scraper/git/GithubScraperTest.java | 70 ++ .../rsd/scraper/git/GitlabScraperIT.java | 52 + .../scraper/git/PostgrestConnectorTest.java | 113 +++ .../GitHubSpdxLicenseRepositoryTest.java | 82 ++ .../scraper/nassa/NassaSoftwareEntryTest.java | 63 ++ .../scrapers/CranScraperTest.java | 53 + .../scrapers/CratesScraperTest.java | 33 + .../scrapers/GoScraperTest.java | 35 + .../scrapers/MavenScraperTest.java | 52 + .../rsd/scraper/ror/RorIdTest.java | 85 ++ .../ror/RorPostgrestConnectorTest.java | 32 + .../rsd/scraper/ror/RorScraperTest.java | 46 + 189 files changed, 21713 insertions(+), 18 deletions(-) create mode 100644 authentication/.editorconfig create mode 100644 authentication/Dockerfile create mode 100644 authentication/README.md create mode 100644 authentication/pom.xml create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/Account.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/AccountInfo.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/AzureLogin.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/Config.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/HelmholtzIdLogin.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/JwtCreator.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/JwtVerifier.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/LinkedinLogin.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/Login.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/Main.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/OpenIdInfo.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/OpenidProvider.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/OpenidProviderAccessMethod.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/OrcidLogin.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/PostgrestAccount.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/QueryParameterBuilder.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdAccountInviteException.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdAuthenticationException.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdProviderData.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdProviders.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdResponseException.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/SurfconextLogin.java create mode 100644 authentication/src/main/java/nl/esciencecenter/rsd/authentication/Utils.java create mode 100644 authentication/src/main/resources/logback.xml create mode 100644 authentication/src/test/java/nl/esciencecenter/rsd/authentication/HelmholtzIdLoginTest.java create mode 100644 authentication/src/test/java/nl/esciencecenter/rsd/authentication/MainTest.java create mode 100644 authentication/src/test/java/nl/esciencecenter/rsd/authentication/PostgrestAccountTest.java create mode 100644 authentication/src/test/java/nl/esciencecenter/rsd/authentication/QueryParameterBuilderTest.java create mode 100644 authentication/src/test/java/nl/esciencecenter/rsd/authentication/RsdProvidersTest.java create mode 100644 authentication/src/test/java/nl/esciencecenter/rsd/authentication/UtilsTest.java create mode 100644 backend-postgrest/Dockerfile create mode 100644 background-services/Dockerfile create mode 100644 background-services/README.md create mode 100644 background-services/pom.xml create mode 100644 background-services/src/main/java/nl/esciencecenter/rsd/MainDatabase.java create mode 100644 background-services/src/main/resources/logback.xml create mode 100644 database/.editorconfig create mode 100644 database/000-fill-authenticator-password.sh create mode 100644 database/001-setup-basic-roles.sql create mode 100644 database/002-create-account-table.sql create mode 100644 database/003-create-image-table.sql create mode 100644 database/004-create-software-table.sql create mode 100644 database/005-create-relations-for-software.sql create mode 100644 database/006-create-project-table.sql create mode 100644 database/007-create-relations-for-projects.sql create mode 100644 database/008-create-community-table.sql create mode 100644 database/010-create-research-domain.sql create mode 100644 database/011-create-mention-table.sql create mode 100644 database/012-create-release-table.sql create mode 100644 database/013-create-organisation-table.sql create mode 100644 database/014-create-keyword-and-category.sql create mode 100644 database/015-inter-relation-tables.sql create mode 100644 database/016-create-maintainer-tables.sql create mode 100644 database/017-meta-pages.sql create mode 100644 database/018-create-backend-logs-table.sql create mode 100644 database/020-row-level-security.sql create mode 100644 database/021-create-global-triggers.sql create mode 100644 database/022-create-maintenance-tables.sql create mode 100644 database/023-news.sql create mode 100644 database/024-remote-rsd.sql create mode 100644 database/025-rsd-info.sql create mode 100644 database/026-user-profile.sql create mode 100644 database/100-create-api-views.sql create mode 100644 database/102-create-account-functions.sql create mode 100644 database/104-person-views.sql create mode 100644 database/104-software-views.sql create mode 100644 database/106-project-views.sql create mode 100644 database/108-community-views.sql create mode 100644 database/109-category-functions.sql create mode 100644 database/113-organisation-views.sql create mode 100644 database/114-keyword-category-views.sql create mode 100644 database/118-backend-logs-views.sql create mode 100644 database/124-aggregated-software-views.sql create mode 100644 database/125-global-search.sql create mode 100644 database/126-public-persons-overview.sql create mode 100644 database/201-nassa-import.sql create mode 100644 database/999-add-categories.sql.example create mode 100644 database/Dockerfile create mode 100644 database/sonar-project.properties create mode 100644 scrapers/.editorconfig create mode 100644 scrapers/Dockerfile create mode 100644 scrapers/README.md create mode 100644 scrapers/jobs.cron create mode 100644 scrapers/pom.xml create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/BibtexParser.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Config.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/InvalidInputException.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdRateLimitException.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdResponseException.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Utils.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/MainAggregator.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/PostgrestConnector.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdConnector.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdData.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CitationData.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CrossrefMention.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataCiteReleaseRepository.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/Doi.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ExternalMentionRecord.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainCitations.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainMentions.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainReleases.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MentionType.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnector.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenalexId.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestCitationRepository.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestMentionRepository.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestReleaseRepository.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ReleaseData.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionIds.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionRecord.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitData.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitDatabaseData.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryData.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryDataWithHistory.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CodePlatformProvider.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitData.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeek.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/ContributorDatabaseData.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/FourTuGitScraper.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitScraper.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GithubScraper.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitlabScraper.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/LanguagesData.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainBasicData.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainCommits.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainContributors.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainProgrammingLanguages.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnector.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepository.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/SpdxLicense.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/MainNassa.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaReadmeParser.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntry.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/RsdNassaParseException.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/BasicPackageManagerData.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/MainPackageManager.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PackageManagerType.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PostgrestConnector.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/AnacondaScraper.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraper.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraper.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/DockerHubScraper.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/FourTuScraper.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraper.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraper.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/NpmScraper.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PackageManagerScraper.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PixiScraper.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PypiScraper.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/MainRor.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/OrganisationDatabaseData.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorData.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorId.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnector.java create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorScraper.java create mode 100644 scrapers/src/main/resources/logback.xml create mode 100644 scrapers/src/main/resources/oaipmh.py create mode 100644 scrapers/src/main/resources/releases.py create mode 100644 scrapers/src/main/resources/requirements.txt create mode 100644 scrapers/src/main/resources/zotero.py create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/BibtexParserTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsIT.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepositoryTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DoiTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/MainMentionsTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnectorTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenalexIdTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeekTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GithubScraperIT.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GithubScraperTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GitlabScraperIT.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnectorTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepositoryTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntryTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraperTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraperTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraperTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraperTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorIdTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnectorTest.java create mode 100644 scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorScraperTest.java diff --git a/authentication/.editorconfig b/authentication/.editorconfig new file mode 100644 index 0000000..f4b6b5f --- /dev/null +++ b/authentication/.editorconfig @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2024 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +# EditorConfig is awesome: https://editorconfig.org + +# Unix-style newlines with a newline ending every file and tab indentation +[**] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true +indent_style = tab + +[**.md] +indent_style = unset diff --git a/authentication/Dockerfile b/authentication/Dockerfile new file mode 100644 index 0000000..9e3286d --- /dev/null +++ b/authentication/Dockerfile @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: 2021 - 2024 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2021 - 2024 Netherlands eScience Center +# SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +# SPDX-FileCopyrightText: 2022 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +# SPDX-FileCopyrightText: 2022 Matthias Rüster (GFZ) +# SPDX-FileCopyrightText: 2022 dv4all +# +# SPDX-License-Identifier: Apache-2.0 + +FROM maven:3.9.8-eclipse-temurin-21 AS builder +WORKDIR /usr/mymaven +RUN mkdir src +COPY pom.xml . +RUN mvn dependency:go-offline +COPY ./src ./src +RUN mvn package --offline + +FROM eclipse-temurin:21-jre-jammy +WORKDIR /usr/myjava +COPY --from=builder /usr/mymaven/target/*-jar-with-dependencies.jar auth.jar +CMD java -cp auth.jar nl.esciencecenter.rsd.authentication.Main diff --git a/authentication/README.md b/authentication/README.md new file mode 100644 index 0000000..ee4192b --- /dev/null +++ b/authentication/README.md @@ -0,0 +1,47 @@ + + +# Authentication module + +[![Coverage](https://sonarcloud.io/api/project_badges/measure?project=nl.research-software%3Aauthentication&metric=coverage)](https://sonarcloud.io/summary/overall?id=nl.research-software%3Aauthentication) + +This module handles authentication from third parties using oAuth2 and OpenID. + +## Environment variables +Check `.env.example` to see which environment variables are needed. In particular, look for the env variable `RSD_ENVIRONMENT` to allow for easy admin creation when developing and testing out the RSD (this is not safe for production!). + +## Contributing + +This module uses [EditorConfig](https://editorconfig.org/) for basic formatting. Please check if your editor [already supports EditorConfig](https://editorconfig.org/#pre-installed) or if you need to [install a plugin](https://editorconfig.org/#download). A GitHub workflow is run on every PR to check if any files violate the formatting settings. + +## Developing locally +If you want to develop and run the auth module locally, i.e. outside of Docker, you have to make two changes to files tracked by Git. +1. In `docker-compose.yml`, add the following lines to the `nginx` service: +```yml +extra_hosts: + - "host.docker.internal:host-gateway" +``` +2. In `nginx.conf`, replace `server auth:7000;` with `server host.docker.internal:7000;` +3. (Optional) For refreshing your tokens to work, set `RSD_AUTH_URL=http://nginx/auth` in your `.env`. +4. (Optional) Allow TCP traffic on port 7000 of your firewall if signing in seems to hang forever or if you get `504 Gateway Timeout` responses. + +Remember to undo these changes before committing! + +It is recommended to use the [envFile plugin](https://plugins.jetbrains.com/plugin/7861-envfile) of IntelliJ IDEA to load your `.env` file. +Furthermore, set the value of `POSTGREST_URL` to `http://localhost/api/v1`. + +## Running the tests + +You can run the tests by executing the docker build in this directory: + +```bash +docker build -t removeme . +# resulting image can be deleted afterwards: +# docker image rm removeme +``` diff --git a/authentication/pom.xml b/authentication/pom.xml new file mode 100644 index 0000000..c4d76ac --- /dev/null +++ b/authentication/pom.xml @@ -0,0 +1,215 @@ + + + + + + + + 4.0.0 + + nl.research-software + authentication + 1.0-SNAPSHOT + + + UTF-8 + 21 + 21 + research-software-directory + https://sonarcloud.io + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.8.1 + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.7.1 + + + + jar-with-dependencies + + + + + + package + + single + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.12 + + + prepare-agent + + prepare-agent + + + + report + + report + + + + XML + + + + + + + + + + + + + io.javalin + javalin + 6.4.0 + + + + + org.jetbrains + annotations + 26.0.1 + + + + + org.slf4j + slf4j-simple + 2.0.16 + + + + + com.auth0 + java-jwt + 4.5.0 + + + + + com.google.code.gson + gson + 2.11.0 + + + + + com.nimbusds + oauth2-oidc-sdk + 11.10.1 + + + + + org.slf4j + slf4j-api + 2.0.16 + + + + + ch.qos.logback + logback-classic + 1.5.12 + + + + + org.junit.jupiter + junit-jupiter + test + + + + + org.junit.platform + junit-platform-launcher + test + + + + + org.apache.maven.surefire + surefire-junit-platform + 3.2.5 + test + + + + + org.mockito + mockito-inline + 5.2.0 + test + + + + + net.bytebuddy + byte-buddy + 1.14.18 + + + + + net.bytebuddy + byte-buddy-agent + 1.14.18 + test + + + + + + + + + org.junit + junit-bom + 5.10.3 + pom + import + + + + diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/Account.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/Account.java new file mode 100644 index 0000000..9ca5272 --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/Account.java @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2022 - 2023 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2023 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import java.io.IOException; + +public interface Account { + + AccountInfo account(OpenIdInfo openIdInfo, OpenidProvider provider) throws IOException, InterruptedException; +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/AccountInfo.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/AccountInfo.java new file mode 100644 index 0000000..8956540 --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/AccountInfo.java @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2022 - 2023 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2023 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +public record AccountInfo( + UUID account, + String name, + boolean isAdmin, + Map> data +) { +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/AzureLogin.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/AzureLogin.java new file mode 100644 index 0000000..dddf41c --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/AzureLogin.java @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.interfaces.DecodedJWT; +import com.google.gson.JsonParser; + +import java.io.IOException; +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +// This authentication provider was developed to work with the Imperial College Azure +// tenant. In principle it should be generic enough to work with a registered +// application from any Azure Active Directory tenant however this has not been tested +// and changes may be needed to further generalise it. + +public class AzureLogin implements Login { + + private final String code; + private final String redirectUrl; + + public AzureLogin(String code, String redirectUrl) { + this.code = Objects.requireNonNull(code); + this.redirectUrl = Objects.requireNonNull(redirectUrl); + } + + @Override + public OpenIdInfo openidInfo() throws IOException, InterruptedException, RsdResponseException { + Map form = createForm(); + String tokenResponse = getTokensFromAzureconext(form); + String idToken = extractIdToken(tokenResponse); + DecodedJWT idJwt = JWT.decode(idToken); + String subject = idJwt.getSubject(); + String email = idJwt.getClaim("email").asString(); + String name = idJwt.getClaim("name").asString(); + Map> emptyData = Collections.emptyMap(); + return new OpenIdInfo(subject, name, email, Config.azureOrganisation(), emptyData); + } + + private Map createForm() { + Map form = new HashMap<>(); + form.put("code", code); + form.put("grant_type", "authorization_code"); + form.put("redirect_uri", redirectUrl); + form.put("scope", "openid"); + form.put("client_id", Config.azureClientId()); + form.put("client_secret", Config.azureClientSecret()); + return form; + } + + private String getTokensFromAzureconext(Map form) throws IOException, InterruptedException, RsdResponseException { + URI tokenEndpoint = Utils.getTokenUrlFromWellKnownUrl(URI.create(Config.azureWellknown())); + return Utils.postForm(tokenEndpoint, form); + } + + private String extractIdToken(String response) { + return JsonParser.parseString(response).getAsJsonObject().getAsJsonPrimitive("id_token").getAsString(); + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/Config.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/Config.java new file mode 100644 index 0000000..169beae --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/Config.java @@ -0,0 +1,184 @@ +// SPDX-FileCopyrightText: 2022 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2022 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +// SPDX-FileCopyrightText: 2022 Matthias Rüster (GFZ) +// SPDX-FileCopyrightText: 2022 dv4all +// SPDX-FileCopyrightText: 2023 - 2025 Christian Meeßen (GFZ) +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collection; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; + +public class Config { + + private Config() { + } + + public static String jwtSigningSecret() { + return System.getenv("PGRST_JWT_SECRET"); + } + + private static Collection rsdAuthCoupleProviders() { + return Optional.ofNullable(System.getenv("RSD_AUTH_COUPLE_PROVIDERS")) + .map(String::toUpperCase) + .map(s -> s.split(";")) + .map(Set::of) + .orElse(Collections.emptySet()); + } + + public static boolean isDevEnv() { + try { + return "dev".equals(System.getenv("RSD_ENVIRONMENT")); + } catch (RuntimeException e) { + return false; + } + } + + public static URI wellKnownUrl(OpenidProvider provider) throws URISyntaxException { + String rawUrl = switch (provider) { + case local -> null; + case surfconext -> surfconextWellknown(); + case helmholtz -> helmholtzIdWellknown(); + case orcid -> orcidWellknown(); + case azure -> azureWellknown(); + case linkedin -> linkedinWellknown(); + }; + + if (rawUrl == null) { + return null; + } + + return new URI(rawUrl); + } + + public static boolean isOrcidCouplingEnabled() { + return rsdAuthCoupleProviders().contains("ORCID"); + } + + public static boolean isLinkedinCouplingEnabled() { + return rsdAuthCoupleProviders().contains("LINKEDIN"); + } + + public static String backendBaseUrl() { + return System.getenv("POSTGREST_URL"); + } + + + // SURFconext + public static String surfconextRedirect() { + return System.getenv("SURFCONEXT_REDIRECT"); + } + + public static String surfconextClientId() { + return System.getenv("SURFCONEXT_CLIENT_ID"); + } + + public static String surfconextWellknown() { + return System.getenv("SURFCONEXT_WELL_KNOWN_URL"); + } + + public static String surfconextClientSecret() { + return System.getenv("AUTH_SURFCONEXT_CLIENT_SECRET"); + } + + + // Helmholtz ID + public static String helmholtzIdRedirect() { + return System.getenv("HELMHOLTZID_REDIRECT"); + } + + public static String helmholtzIdClientId() { + return System.getenv("HELMHOLTZID_CLIENT_ID"); + } + + public static String helmholtzIdWellknown() { + return System.getenv("HELMHOLTZID_WELL_KNOWN_URL"); + } + + public static String helmholtzIdClientSecret() { + return System.getenv("AUTH_HELMHOLTZID_CLIENT_SECRET"); + } + + public static String helmholtzIdScopes() { + return System.getenv("HELMHOLTZID_SCOPES"); + } + + public static boolean helmholtzIdAllowExternalUsers() { + return Boolean.parseBoolean( + System.getenv("HELMHOLTZID_ALLOW_EXTERNAL_USERS") + ); + } + + // ORCID + public static String orcidRedirect() { + return System.getenv("ORCID_REDIRECT"); + } + + public static String orcidRedirectCouple() { + return System.getenv("ORCID_REDIRECT_COUPLE"); + } + + public static String orcidClientId() { + return System.getenv("ORCID_CLIENT_ID"); + } + + public static String orcidWellknown() { + return System.getenv("ORCID_WELL_KNOWN_URL"); + } + + public static String orcidClientSecret() { + return System.getenv("AUTH_ORCID_CLIENT_SECRET"); + } + + + // Azure Active Directory + public static String azureRedirect() { + return System.getenv("AZURE_REDIRECT"); + } + + public static String azureClientId() { + return System.getenv("AZURE_CLIENT_ID"); + } + + public static String azureWellknown() { + return System.getenv("AZURE_WELL_KNOWN_URL"); + } + + public static String azureClientSecret() { + return System.getenv("AUTH_AZURE_CLIENT_SECRET"); + } + + public static String azureOrganisation() { + return System.getenv("AZURE_ORGANISATION"); + } + + + // LinkedIn + public static String linkedinRedirect() { + return System.getenv("LINKEDIN_REDIRECT"); + } + + public static String linkedinRedirectCouple() { + return System.getenv("LINKEDIN_REDIRECT_COUPLE"); + } + + public static String linkedinClientId() { + return System.getenv("LINKEDIN_CLIENT_ID"); + } + + public static String linkedinWellknown() { + return System.getenv("LINKEDIN_WELL_KNOWN_URL"); + } + + public static String linkedinClientSecret() { + return System.getenv("AUTH_LINKEDIN_CLIENT_SECRET"); + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/HelmholtzIdLogin.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/HelmholtzIdLogin.java new file mode 100644 index 0000000..45d399e --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/HelmholtzIdLogin.java @@ -0,0 +1,228 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2022 Matthias Rüster (GFZ) +// SPDX-FileCopyrightText: 2023 - 2025 Christian Meeßen (GFZ) +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; +import com.nimbusds.oauth2.sdk.AccessTokenResponse; +import com.nimbusds.oauth2.sdk.AuthorizationCode; +import com.nimbusds.oauth2.sdk.AuthorizationCodeGrant; +import com.nimbusds.oauth2.sdk.AuthorizationGrant; +import com.nimbusds.oauth2.sdk.Scope; +import com.nimbusds.oauth2.sdk.TokenErrorResponse; +import com.nimbusds.oauth2.sdk.TokenRequest; +import com.nimbusds.oauth2.sdk.TokenResponse; +import com.nimbusds.oauth2.sdk.auth.ClientAuthentication; +import com.nimbusds.oauth2.sdk.auth.ClientSecretBasic; +import com.nimbusds.oauth2.sdk.auth.Secret; +import com.nimbusds.oauth2.sdk.http.HTTPResponse; +import com.nimbusds.oauth2.sdk.id.ClientID; +import com.nimbusds.oauth2.sdk.token.BearerAccessToken; +import com.nimbusds.openid.connect.sdk.UserInfoRequest; +import com.nimbusds.openid.connect.sdk.UserInfoResponse; +import com.nimbusds.openid.connect.sdk.claims.UserInfo; +import net.minidev.json.JSONArray; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class HelmholtzIdLogin implements Login { + + private final String code; + private final String redirectUrl; + static final String DEFAULT_ORGANISATION = "Helmholtz"; + + // See https://hifis.net/doc/helmholtz-aai/list-of-vos/#vos-representing-helmholtz-centres + private static final Collection knownHgfOrganisations = Set.of( + "AWI", "CISPA", "DESY", "DKFZ", "DLR", "DZNE", "FZJ", "GEOMAR", "GFZ", "GSI", "hereon", "HMGU", "HZB", "HZDR", "HZI", "KIT", "MDC", "UFZ" + ); + + public HelmholtzIdLogin(String code, String redirectUrl) { + this.code = Objects.requireNonNull(code); + this.redirectUrl = Objects.requireNonNull(redirectUrl); + } + + static String getOrganisationFromEntitlements(JSONArray entitlements) { + if (entitlements == null || entitlements.isEmpty()) { + return null; + } + + String returnOrganisation; + ArrayList organisationsDelivered = new ArrayList<>(); + boolean helmholtzmemberFound = false; + + // Collect all organisations delivered, because the home organisation + // must not be the first one in the list. This assumes that a person + // is only member of one organisation + String organisation; + for (Object element : entitlements.toArray()) { + organisation = null; + String ent = element.toString(); + + // we expect this for logins from Helmholtz centres + // see https://hifis.net/doc/helmholtz-aai/list-of-vos/#vos-representing-helmholtz-centres + if (ent.matches("urn:geant:helmholtz\\.de:group:Helmholtz-member.*")) { + helmholtzmemberFound = true; + continue; + } + + // example: urn:geant:helmholtz.de:group:GFZ#login.helmholtz.de + if (ent.matches("urn:geant:helmholtz\\.de:group:.*")) { + String withoutHash = ent; + + // remove everything behind the hash + if (ent.contains("#")) { + String[] splitHash = ent.split("#"); + + if (splitHash.length != 2) { + // unknown format + continue; + } + + withoutHash = splitHash[0]; + } + + // urn:geant:helmholtz.de:group:GFZ + String[] splitGroup = withoutHash.split(":"); + + if (splitGroup.length != 5) { + // unknown format + continue; + } + + // get organisation from last element + organisation = splitGroup[splitGroup.length - 1]; + organisationsDelivered.add(organisation); + } + } + + if (!helmholtzmemberFound) { + return null; + } + + // Detect whether one of the delivered organisations is in the list of known HGF centres + organisationsDelivered.retainAll(knownHgfOrganisations); + if (organisationsDelivered.isEmpty()) { + // No known HGF organisation could be found + returnOrganisation = DEFAULT_ORGANISATION; + } else { + // Always return the first element in the list, even if there were multiple centres found + returnOrganisation = organisationsDelivered.get(0); + } + + // else: we either return the found the Helmholtz centre name + // or the default organisation + return returnOrganisation; + } + + @Override + public OpenIdInfo openidInfo() throws IOException, InterruptedException { + UserInfo userInfo; + String userinfoUrl; + + /* get the userinfo endpoint URL first */ + HttpRequest request = HttpRequest.newBuilder( + URI.create(Config.helmholtzIdWellknown()) + ) + .header("accept", "application/json") + .build(); + + try (HttpClient client = HttpClient.newHttpClient()) { + HttpResponse response = client.send(request, BodyHandlers.ofString()); + JsonElement resp = JsonParser.parseString(response.body()); + userinfoUrl = resp.getAsJsonObject().getAsJsonPrimitive("userinfo_endpoint").getAsString(); + } + + try { + AuthorizationCode authcode = new AuthorizationCode(code); + URI callback = new URI(redirectUrl); + AuthorizationGrant codeGrant = new AuthorizationCodeGrant(authcode, callback); + ClientID clientID = new ClientID(Config.helmholtzIdClientId()); + Secret clientSecret = new Secret(Config.helmholtzIdClientSecret()); + ClientAuthentication clientAuth = new ClientSecretBasic(clientID, clientSecret); + URI tokenEndpoint = Utils.getTokenUrlFromWellKnownUrl(URI.create(Config.helmholtzIdWellknown())); + + Scope scopes = new Scope(); + + for (String scope : Config.helmholtzIdScopes().split(" ")) { + scopes.add(scope); + } + + TokenRequest tokenRequest = new TokenRequest( + tokenEndpoint, clientAuth, codeGrant, scopes + ); + TokenResponse response = TokenResponse.parse(tokenRequest.toHTTPRequest().send()); + + if (!response.indicatesSuccess()) { + TokenErrorResponse errorResponse = response.toErrorResponse(); + throw new Exception(errorResponse.getErrorObject().getDescription()); + } + + AccessTokenResponse successResponse = response.toSuccessResponse(); + + URI userInfoEndpoint = new URI(userinfoUrl); + BearerAccessToken token = successResponse.getTokens().getBearerAccessToken(); + HTTPResponse httpResponse = new UserInfoRequest(userInfoEndpoint, token) + .toHTTPRequest() + .send(); + + UserInfoResponse userInfoResponse = UserInfoResponse.parse(httpResponse); + + if (!userInfoResponse.indicatesSuccess()) { + throw new Exception("userinfo response was not successful"); + } + + userInfo = userInfoResponse.toSuccessResponse().getUserInfo(); + } catch (InterruptedException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException(e); + } + + JSONArray entitlements = new JSONArray(); + Object edupersonClaim = userInfo.getClaim("eduperson_entitlement"); + if (edupersonClaim instanceof JSONArray jsonArray) { + entitlements = jsonArray; + } else if (edupersonClaim instanceof String) { + entitlements.appendElement(edupersonClaim); + } else if (edupersonClaim == null) { + entitlements.appendElement(""); + } else { + throw new RuntimeException("Unexpected return type of eduperson_entitlement."); + } + String organisation = getOrganisationFromEntitlements(entitlements); + + List eduPersonEntitlements = new ArrayList<>(); + for (int i=0; i> data = new HashMap<>(); + data.put("eduPersonEntitlements", eduPersonEntitlements); + + return new OpenIdInfo( + userInfo.getSubject().toString(), + userInfo.getName(), + userInfo.getEmailAddress(), + organisation, + data + ); + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/JwtCreator.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/JwtCreator.java new file mode 100644 index 0000000..81b31e7 --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/JwtCreator.java @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: 2022 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +// SPDX-FileCopyrightText: 2022 dv4all +// SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.algorithms.Algorithm; +import com.auth0.jwt.interfaces.DecodedJWT; +import com.google.gson.Gson; + +import java.io.IOException; +import java.util.Date; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; + +public class JwtCreator { + + static final long ONE_HOUR_IN_MILLISECONDS = 3600_000L; // 60 * 60 * 1000 + private final Algorithm signingAlgorithm; + private static final String RSD_ADMIN_ROLE = "rsd_admin"; + private static final String RSD_USER_ROLE = "rsd_user"; + + public JwtCreator(String signingSecret) { + Objects.requireNonNull(signingSecret); + this.signingAlgorithm = Algorithm.HMAC256(signingSecret); + } + + String createUserJwt(AccountInfo accountInfo) { + return JWT.create() + .withClaim("iss", "rsd_auth") + .withClaim("role", accountInfo.isAdmin() ? RSD_ADMIN_ROLE : RSD_USER_ROLE) + .withClaim("account", accountInfo.account().toString()) + .withClaim("name", accountInfo.name()) + .withClaim("data", accountInfo.data()) + .withExpiresAt(new Date(System.currentTimeMillis() + ONE_HOUR_IN_MILLISECONDS)) + .sign(signingAlgorithm); + } + + + String createAdminJwt() { + return JWT.create() + .withClaim("iss", "rsd_auth") + .withClaim("role", RSD_ADMIN_ROLE) + .withExpiresAt(new Date(System.currentTimeMillis() + ONE_HOUR_IN_MILLISECONDS)) + .sign(signingAlgorithm); + } + + String refreshToken(String token) throws IOException, InterruptedException { + DecodedJWT oldJwt = JWT.decode(token); + UUID accountId = UUID.fromString(oldJwt.getClaim("account").asString()); + boolean isAdmin = new PostgrestAccount(Config.backendBaseUrl()).isAdmin(accountId); + String payloadEncoded = oldJwt.getPayload(); + String payloadDecoded = Main.decode(payloadEncoded); + Gson gson = new Gson(); + Map claimsMap = gson.>fromJson(payloadDecoded, Map.class); + return JWT.create() + .withPayload(claimsMap) + .withClaim("role", isAdmin ? RSD_ADMIN_ROLE : RSD_USER_ROLE) + .withExpiresAt(new Date(System.currentTimeMillis() + ONE_HOUR_IN_MILLISECONDS)) + .sign(signingAlgorithm); + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/JwtVerifier.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/JwtVerifier.java new file mode 100644 index 0000000..36af2cd --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/JwtVerifier.java @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: 2022 - 2023 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2023 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.JWTVerifier; +import com.auth0.jwt.algorithms.Algorithm; +import com.auth0.jwt.exceptions.JWTVerificationException; +import com.auth0.jwt.interfaces.DecodedJWT; + +import java.util.Objects; + +public class JwtVerifier { + + private final String signingSecret; + + public JwtVerifier(String signingSecret) { + this.signingSecret = Objects.requireNonNull(signingSecret); + } + + DecodedJWT verify(String token) { + if (token == null) { + throw new JWTVerificationException("Token was null"); + } + Algorithm signingAlgorithm = Algorithm.HMAC256(signingSecret); + JWTVerifier verifier = JWT.require(signingAlgorithm).build(); + return verifier.verify(token); + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/LinkedinLogin.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/LinkedinLogin.java new file mode 100644 index 0000000..417cd68 --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/LinkedinLogin.java @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.interfaces.DecodedJWT; +import com.google.gson.JsonParser; + +import java.io.IOException; +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class LinkedinLogin implements Login { + + private final String code; + private final String redirectUrl; + + public LinkedinLogin(String code, String redirectUrl) { + this.code = Objects.requireNonNull(code); + this.redirectUrl = Objects.requireNonNull(redirectUrl); + } + + @Override + public OpenIdInfo openidInfo() throws IOException, InterruptedException, RsdResponseException { + Map form = createForm(); + String tokenResponse = getTokensFromLinkedin(form); + String idToken = extractIdToken(tokenResponse); + + DecodedJWT idJwt = JWT.decode(idToken); + String subject = idJwt.getSubject(); + String name = idJwt.getClaim("name").asString(); + if (name == null) { + String givenName = idJwt.getClaim("given_name").asString(); + String familyName = idJwt.getClaim("family_name").asString(); + if (givenName != null && familyName != null) name = givenName + " " + familyName; + else if (familyName != null) name = familyName; + else if (givenName != null) name = givenName; + } + String email = idJwt.getClaim("email").asString(); + Map> emptyData = Collections.emptyMap(); + return new OpenIdInfo(subject, name, email, null, emptyData); + } + + private Map createForm() { + Map form = new HashMap<>(); + form.put("code", code); + form.put("grant_type", "authorization_code"); + form.put("redirect_uri", redirectUrl); + form.put("client_id", Config.linkedinClientId()); + form.put("client_secret", Config.linkedinClientSecret()); + return form; + } + + private String getTokensFromLinkedin(Map form) throws IOException, InterruptedException, RsdResponseException { + URI tokenEndpoint = Utils.getTokenUrlFromWellKnownUrl(URI.create(Config.linkedinWellknown())); + return Utils.postForm(tokenEndpoint, form); + } + + private String extractIdToken(String response) { + return JsonParser.parseString(response).getAsJsonObject().getAsJsonPrimitive("id_token").getAsString(); + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/Login.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/Login.java new file mode 100644 index 0000000..318a893 --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/Login.java @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2021 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2021 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import java.io.IOException; + +public interface Login { + + OpenIdInfo openidInfo() throws IOException, InterruptedException, RsdResponseException; +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/Main.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/Main.java new file mode 100644 index 0000000..9e713e0 --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/Main.java @@ -0,0 +1,389 @@ +// SPDX-FileCopyrightText: 2021 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2021 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +// SPDX-FileCopyrightText: 2022 Matthias Rüster (GFZ) +// SPDX-FileCopyrightText: 2022 dv4all +// SPDX-FileCopyrightText: 2023 - 2025 Christian Meeßen (GFZ) +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import com.auth0.jwt.exceptions.JWTVerificationException; +import com.auth0.jwt.interfaces.DecodedJWT; +import io.javalin.Javalin; +import io.javalin.http.Context; +import io.javalin.http.HttpStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.Duration; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +public class Main { + private static final Duration AUTH_COOKIE_DURATION = Duration.ofHours(1); + private static final Duration FAILURE_COOKIE_DURATION = Duration.ofMinutes(1); + private static final String INVITE_COOKIE_NAME = "rsd_invite_id"; + + private static final Logger LOGGER = LoggerFactory.getLogger(Main.class); + private static final String LOGIN_FAILED_PATH = "/login/failed"; + + public static boolean idUserIsHelmholtzMember(OpenIdInfo helmholtzInfo) { + if (helmholtzInfo.organisation() == null) { + return false; + } else { + return true; + } + } + + public static void main(String[] args) { + RsdProviders rsdProviders = new RsdProviders(); + Javalin app = Javalin.create(c -> c.useVirtualThreads = false).start(7000); + + app.afterMatched("/login/*", ctx -> { + ctx.removeCookie(INVITE_COOKIE_NAME, "/auth"); + }); + + app.get("/", ctx -> ctx.json("{\"Module\": \"rsd/auth\", \"Status\": \"live\"}")); + + app.get("/providers", ctx -> { + ctx.json(rsdProviders.activeProvidersAsJson()); + }); + + app.post("/login/local", ctx -> { + OpenidProvider localProvider = OpenidProvider.local; + switch (rsdProviders.accessMethodOfProvider(localProvider)) { + case MISCONFIGURED -> { + handleMisconfiguredProvider(ctx, localProvider); + } + case DISABLED -> { + handleDisabledProvider(ctx, localProvider); + } + case INVITE_ONLY -> { + OpenIdInfo localInfo = extractLocalInfo(ctx); + + handleAccountInviteOnly(localInfo, localProvider, ctx); + } + case EVERYONE -> { + OpenIdInfo localInfo = extractLocalInfo(ctx); + + handleAccountEveryoneAllowed(localInfo, localProvider, ctx); + } + } + }); + + app.post("/login/surfconext", ctx -> { + OpenidProvider surfProvider = OpenidProvider.surfconext; + switch (rsdProviders.accessMethodOfProvider(surfProvider)) { + case MISCONFIGURED -> { + handleMisconfiguredProvider(ctx, surfProvider); + } + case DISABLED -> { + handleDisabledProvider(ctx, surfProvider); + } + case INVITE_ONLY -> { + OpenIdInfo surfconextInfo = extractSurfInfo(ctx); + + handleAccountInviteOnly(surfconextInfo, surfProvider, ctx); + } + case EVERYONE -> { + OpenIdInfo surfconextInfo = extractSurfInfo(ctx); + + handleAccountEveryoneAllowed(surfconextInfo, surfProvider, ctx); + } + } + }); + + app.get("/login/helmholtzid", ctx -> { + OpenidProvider helmholtzProvider = OpenidProvider.helmholtz; + switch (rsdProviders.accessMethodOfProvider(helmholtzProvider)) { + case MISCONFIGURED -> { + handleMisconfiguredProvider(ctx, helmholtzProvider); + } + case DISABLED -> { + handleDisabledProvider(ctx, helmholtzProvider); + } + case INVITE_ONLY -> { + OpenIdInfo helmholtzInfo = extractHelmholtzInfo(ctx); + + handleAccountInviteOnly(helmholtzInfo, helmholtzProvider, ctx); + } + case EVERYONE -> { + OpenIdInfo helmholtzInfo = extractHelmholtzInfo(ctx); + + if (!idUserIsHelmholtzMember(helmholtzInfo) && !Config.helmholtzIdAllowExternalUsers()) { + // If no external members can login by default, access can be granted via invites + handleAccountInviteOnly(helmholtzInfo, helmholtzProvider, ctx); + } else { + handleAccountEveryoneAllowed(helmholtzInfo, helmholtzProvider, ctx); + } + } + } + }); + + app.get("/login/orcid", ctx -> { + OpenidProvider orcidProvider = OpenidProvider.orcid; + switch (rsdProviders.accessMethodOfProvider(orcidProvider)) { + case MISCONFIGURED -> { + handleMisconfiguredProvider(ctx, orcidProvider); + } + case DISABLED -> { + handleDisabledProvider(ctx, orcidProvider); + } + case INVITE_ONLY -> { + OpenIdInfo orcidInfo = extractOrcidInfo(ctx); + + handleAccountInviteOnly(orcidInfo, orcidProvider, ctx); + } + case EVERYONE -> { + OpenIdInfo orcidInfo = extractOrcidInfo(ctx); + + handleAccountEveryoneAllowed(orcidInfo, orcidProvider, ctx); + } + } + }); + + if (Config.isOrcidCouplingEnabled()) { + app.get("/couple/orcid", ctx -> { + String code = ctx.queryParam("code"); + String redirectUrl = Config.orcidRedirectCouple(); + OpenIdInfo orcidInfo = new OrcidLogin(code, redirectUrl).openidInfo(); + + String tokenToVerify = ctx.cookie("rsd_token"); + String signingSecret = Config.jwtSigningSecret(); + JwtVerifier verifier = new JwtVerifier(signingSecret); + DecodedJWT decodedJWT = verifier.verify(tokenToVerify); + UUID accountId = UUID.fromString(decodedJWT.getClaim("account").asString()); + + new PostgrestAccount(Config.backendBaseUrl()).coupleLogin(accountId, orcidInfo, OpenidProvider.orcid); + + setRedirectFromCookie(ctx); + }); + } + + app.get("/login/azure", ctx -> { + OpenidProvider azureProvider = OpenidProvider.azure; + switch (rsdProviders.accessMethodOfProvider(azureProvider)) { + case MISCONFIGURED -> { + handleMisconfiguredProvider(ctx, azureProvider); + } + case DISABLED -> { + handleDisabledProvider(ctx, azureProvider); + } + case INVITE_ONLY -> { + OpenIdInfo azureInfo = extractAzureInfo(ctx); + + handleAccountInviteOnly(azureInfo, azureProvider, ctx); + } + case EVERYONE -> { + OpenIdInfo azureInfo = extractAzureInfo(ctx); + + handleAccountEveryoneAllowed(azureInfo, azureProvider, ctx); + } + } + }); + + app.get("login/linkedin", ctx -> { + OpenidProvider linkedinProvider = OpenidProvider.linkedin; + switch (rsdProviders.accessMethodOfProvider(linkedinProvider)) { + case MISCONFIGURED -> { + handleMisconfiguredProvider(ctx, linkedinProvider); + } + case DISABLED -> { + handleDisabledProvider(ctx, linkedinProvider); + } + case INVITE_ONLY -> { + OpenIdInfo linkedinInfo = extractLinkedinInfo(ctx); + + handleAccountInviteOnly(linkedinInfo, linkedinProvider, ctx); + } + case EVERYONE -> { + OpenIdInfo linkedinInfo = extractLinkedinInfo(ctx); + + handleAccountEveryoneAllowed(linkedinInfo, linkedinProvider, ctx); + } + } + }); + + if (Config.isLinkedinCouplingEnabled()) { + app.get("/couple/linkedin", ctx -> { + String code = ctx.queryParam("code"); + String redirectUrl = Config.linkedinRedirectCouple(); + OpenIdInfo linkedinInfo = new LinkedinLogin(code, redirectUrl).openidInfo(); + + String tokenToVerify = ctx.cookie("rsd_token"); + String signingSecret = Config.jwtSigningSecret(); + JwtVerifier verifier = new JwtVerifier(signingSecret); + DecodedJWT decodedJWT = verifier.verify(tokenToVerify); + UUID accountId = UUID.fromString(decodedJWT.getClaim("account").asString()); + + new PostgrestAccount(Config.backendBaseUrl()).coupleLogin(accountId, linkedinInfo, OpenidProvider.linkedin); + + setRedirectFromCookie(ctx); + }); + } + + app.get("/refresh", ctx -> { + try { + String tokenToVerify = ctx.cookie("rsd_token"); + String signingSecret = Config.jwtSigningSecret(); + JwtVerifier verifier = new JwtVerifier(signingSecret); + verifier.verify(tokenToVerify); + + JwtCreator jwtCreator = new JwtCreator(signingSecret); + String token = jwtCreator.refreshToken(tokenToVerify); + setJwtCookie(ctx, token); + } catch (RuntimeException ex) { + LOGGER.error("RuntimeException", ex); + ctx.status(400); + ctx.json("{\"message\": \"failed to refresh token\"}"); + } + }); + + app.exception(JWTVerificationException.class, (ex, ctx) -> { + LOGGER.error("JWTVerificationException", ex); + ctx.status(400); + ctx.json("{\"message\": \"invalid JWT\"}"); + }); + + app.exception(RsdAuthenticationException.class, (ex, ctx) -> { + setLoginFailureCookie(ctx, ex.getMessage()); + ctx.redirect(LOGIN_FAILED_PATH, HttpStatus.SEE_OTHER); + }); + + app.exception(RsdAccountInviteException.class, (ex, ctx) -> { + setLoginFailureCookie(ctx, ex.getMessage()); + ctx.redirect(LOGIN_FAILED_PATH, HttpStatus.SEE_OTHER); + }); + + app.exception(RuntimeException.class, (ex, ctx) -> { + LOGGER.error("RuntimeException", ex); + setLoginFailureCookie(ctx, "Something unexpected went wrong, please try again or contact us."); + ctx.redirect(LOGIN_FAILED_PATH, HttpStatus.SEE_OTHER); + }); + + app.exception(Exception.class, (ex, ctx) -> { + LOGGER.error("Exception", ex); + setLoginFailureCookie(ctx, "Something unexpected went wrong, please try again or contact us."); + ctx.redirect(LOGIN_FAILED_PATH, HttpStatus.SEE_OTHER); + }); + } + + static OpenIdInfo extractLocalInfo(Context ctx) { + String sub = ctx.formParam("sub"); + if (sub == null || sub.isBlank()) throw new RuntimeException("Please provide a username"); + String name = sub; + String email = sub + "@example.com"; + String organisation = "Example organisation"; + Map> emptyData = Collections.emptyMap(); + return new OpenIdInfo(sub, name, email, organisation, emptyData); + } + + static OpenIdInfo extractSurfInfo(Context ctx) throws RsdResponseException, IOException, InterruptedException { + String code = ctx.formParam("code"); + String redirectUrl = Config.surfconextRedirect(); + return new SurfconextLogin(code, redirectUrl).openidInfo(); + } + + static OpenIdInfo extractHelmholtzInfo(Context ctx) throws IOException, InterruptedException { + String code = ctx.queryParam("code"); + String redirectUrl = Config.helmholtzIdRedirect(); + return new HelmholtzIdLogin(code, redirectUrl).openidInfo(); + } + + static OpenIdInfo extractOrcidInfo(Context ctx) throws RsdResponseException, IOException, InterruptedException { + String code = ctx.queryParam("code"); + String redirectUrl = Config.orcidRedirect(); + return new OrcidLogin(code, redirectUrl).openidInfo(); + } + + static OpenIdInfo extractAzureInfo(Context ctx) throws RsdResponseException, IOException, InterruptedException { + String code = ctx.queryParam("code"); + String redirectUrl = Config.azureRedirect(); + return new AzureLogin(code, redirectUrl).openidInfo(); + } + + static OpenIdInfo extractLinkedinInfo(Context ctx) throws RsdResponseException, IOException, InterruptedException { + String code = ctx.queryParam("code"); + String redirectUrl = Config.linkedinRedirect(); + return new LinkedinLogin(code, redirectUrl).openidInfo(); + } + + static void handleAccountInviteOnly(OpenIdInfo openIdInfo, OpenidProvider provider, Context ctx) throws IOException, InterruptedException, RsdAccountInviteException { + PostgrestAccount postgrestAccount = new PostgrestAccount(Config.backendBaseUrl()); + Optional optionalAccountInfo = postgrestAccount.getAccountIfExists(openIdInfo, provider); + if (optionalAccountInfo.isPresent()) { + createAndSetToken(ctx, optionalAccountInfo.get()); + } else { + UUID inviteId = getInviteIdFromRequest(ctx); + AccountInfo accountInfo = postgrestAccount.useInviteToCreateAccount(inviteId, openIdInfo, provider); + createAndSetToken(ctx, accountInfo); + } + } + + static void handleAccountEveryoneAllowed(OpenIdInfo openIdInfo, OpenidProvider provider, Context ctx) throws IOException, InterruptedException { + AccountInfo accountInfo = new PostgrestAccount(Config.backendBaseUrl()).account(openIdInfo, provider); + createAndSetToken(ctx, accountInfo); + } + + static void handleDisabledProvider(Context ctx, OpenidProvider provider) { + String message = "The provider \"%s\", is disabled, please try a different provider.".formatted(provider.toUserFriendlyString()); + setLoginFailureCookie(ctx, message); + ctx.redirect(LOGIN_FAILED_PATH, HttpStatus.SEE_OTHER); + } + + static void handleMisconfiguredProvider(Context ctx, OpenidProvider provider) { + String message = "The provider \"%s\", is misconfigured, please contact your RSD admins.".formatted(provider.toUserFriendlyString()); + setLoginFailureCookie(ctx, message); + ctx.redirect(LOGIN_FAILED_PATH, HttpStatus.SEE_OTHER); + } + + static UUID getInviteIdFromRequest(Context ctx) throws RsdAccountInviteException { + String cookie = ctx.cookie(INVITE_COOKIE_NAME); + if (cookie == null) { + throw new RsdAccountInviteException("No invite found to create an account"); + } + + try { + return UUID.fromString(cookie); + } catch (IllegalArgumentException e) { + throw new RsdAccountInviteException("Received invalid invite ID", e); + } + } + + static void createAndSetToken(Context ctx, AccountInfo accountInfo) { + JwtCreator jwtCreator = new JwtCreator(Config.jwtSigningSecret()); + String token = jwtCreator.createUserJwt(accountInfo); + setJwtCookie(ctx, token); + setRedirectFromCookie(ctx); + } + + static void setJwtCookie(Context ctx, String token) { + ctx.header("Set-Cookie", "rsd_token=" + token + "; Secure; HttpOnly; Path=/; SameSite=Lax; Max-Age=" + AUTH_COOKIE_DURATION.toSeconds()); + } + + static void setLoginFailureCookie(Context ctx, String message) { + ctx.header("Set-Cookie", "rsd_login_failure_message=" + message + "; Secure; Path=/login/failed; SameSite=Lax; Max-Age=" + FAILURE_COOKIE_DURATION.toSeconds()); + } + + static void setRedirectFromCookie(Context ctx) { + String returnPath = ctx.cookie("rsd_pathname"); + if (returnPath != null && !returnPath.isBlank()) { + returnPath = returnPath.trim(); + ctx.redirect(returnPath, HttpStatus.SEE_OTHER); + } else { + ctx.redirect("/", HttpStatus.SEE_OTHER); + } + } + + static String decode(String base64UrlEncoded) { + return new String(Base64.getUrlDecoder().decode(base64UrlEncoded)); + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/OpenIdInfo.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/OpenIdInfo.java new file mode 100644 index 0000000..39ae4df --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/OpenIdInfo.java @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import java.util.List; +import java.util.Map; + +public record OpenIdInfo( + String sub, + String name, + String email, + String organisation, + Map> data +) { +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/OpenidProvider.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/OpenidProvider.java new file mode 100644 index 0000000..235fd22 --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/OpenidProvider.java @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2022 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +public enum OpenidProvider { + local, + surfconext, + helmholtz, + orcid, + azure, + linkedin; + + public String toUserFriendlyString() { + return switch (this) { + case local -> "local accounts"; + case surfconext -> "SURFconext"; + case helmholtz -> "Helmholtz"; + case orcid -> "ORCID"; + case azure -> "Azure"; + case linkedin -> "LinkedIn"; + }; + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/OpenidProviderAccessMethod.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/OpenidProviderAccessMethod.java new file mode 100644 index 0000000..0a83ed8 --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/OpenidProviderAccessMethod.java @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +public enum OpenidProviderAccessMethod { + MISCONFIGURED { + @Override + public boolean isActive() { + return false; + } + }, + DISABLED { + @Override + public boolean isActive() { + return false; + } + }, + INVITE_ONLY { + @Override + public boolean isActive() { + return true; + } + }, + EVERYONE { + @Override + public boolean isActive() { + return true; + } + }; + + public abstract boolean isActive(); +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/OrcidLogin.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/OrcidLogin.java new file mode 100644 index 0000000..366fa32 --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/OrcidLogin.java @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.interfaces.DecodedJWT; +import com.google.gson.JsonParser; + +import java.io.IOException; +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class OrcidLogin implements Login { + + private final String code; + private final String redirectUrl; + + public OrcidLogin(String code, String redirectUrl) { + this.code = Objects.requireNonNull(code); + this.redirectUrl = Objects.requireNonNull(redirectUrl); + } + + @Override + public OpenIdInfo openidInfo() throws IOException, InterruptedException, RsdResponseException { + Map form = createForm(); + String tokenResponse = getTokensFromOrcidconext(form); + String idToken = extractIdToken(tokenResponse); + + DecodedJWT idJwt = JWT.decode(idToken); + String subject = idJwt.getSubject(); + String name = idJwt.getClaim("name").asString(); + if (name == null) { + String givenName = idJwt.getClaim("given_name").asString(); + String familyName = idJwt.getClaim("family_name").asString(); + if (givenName != null && familyName != null) name = givenName + " " + familyName; + else if (familyName != null) name = familyName; + else if (givenName != null) name = givenName; + } + Map> emptyData = Collections.emptyMap(); + return new OpenIdInfo(subject, name, null, null, emptyData); + } + + private Map createForm() { + Map form = new HashMap<>(); + form.put("code", code); + form.put("grant_type", "authorization_code"); + form.put("redirect_uri", redirectUrl); + form.put("scope", "openid"); + form.put("client_id", Config.orcidClientId()); + form.put("client_secret", Config.orcidClientSecret()); + return form; + } + + private String getTokensFromOrcidconext(Map form) throws IOException, InterruptedException, RsdResponseException { + URI tokenEndpoint = Utils.getTokenUrlFromWellKnownUrl(URI.create(Config.orcidWellknown())); + return Utils.postForm(tokenEndpoint, form); + } + + private String extractIdToken(String response) { + return JsonParser.parseString(response).getAsJsonObject().getAsJsonPrimitive("id_token").getAsString(); + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/PostgrestAccount.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/PostgrestAccount.java new file mode 100644 index 0000000..4e2ed0f --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/PostgrestAccount.java @@ -0,0 +1,319 @@ +// SPDX-FileCopyrightText: 2022 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import java.io.IOException; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; + +public class PostgrestAccount implements Account { + + private static final String AUTHORIZATION_HEADER_KEY = "Authorization"; + private static final String BEARER_HEADER_VALUE_PREFIX = "bearer"; + private static final String PREFER_HEADER_KEY = "Prefer"; + + private final String backendUri; + + public PostgrestAccount(String backendUri) { + this.backendUri = backendUri; + } + + public Optional getAccountIfExists(OpenIdInfo openIdInfo, OpenidProvider provider) throws IOException, InterruptedException { + Objects.requireNonNull(openIdInfo); + Objects.requireNonNull(provider); + + String subject = openIdInfo.sub(); + String subjectUrlEncoded = URLEncoder.encode(subject, StandardCharsets.UTF_8); + String providerUrlEncoded = URLEncoder.encode(provider.toString(), StandardCharsets.UTF_8); + + // The following URI sees if the login credentials already are tied to an account. + // If yes, it also, by joining through the account table, looks up if the account is an admin. + URI queryUri = URI.create(backendUri + "/login_for_account?select=id,account_id:account,name,account(admin_account(account_id))&sub=eq." + subjectUrlEncoded + "&provider=eq." + providerUrlEncoded); + JwtCreator jwtCreator = new JwtCreator(Config.jwtSigningSecret()); + String token = jwtCreator.createAdminJwt(); + String responseLogin = getAsAdmin(queryUri, token); + JsonArray accountsWithSub = JsonParser.parseString(responseLogin).getAsJsonArray(); + // Because of the UNIQUE(provider, sub) constraint on the login_for_account table, this should never happen + if (accountsWithSub.size() > 1) { + throw new RuntimeException("More than one login for subject " + subject + " exists"); + } + // The credentials are already tied to an account, so we update the login credentials with a possibly new name, email, etc., + // and we return the existing account. + else if (accountsWithSub.size() == 1) { + JsonObject accountInfo = accountsWithSub.get(0).getAsJsonObject(); + UUID id = UUID.fromString(accountInfo.getAsJsonPrimitive("id").getAsString()); + updateLoginForAccount(id, openIdInfo, token); + UUID account = UUID.fromString(accountInfo.getAsJsonPrimitive("account_id").getAsString()); + String name = openIdInfo.name(); + + boolean isAdmin = accountInfo.getAsJsonObject("account").get("admin_account").isJsonObject() + && + accountInfo.getAsJsonObject("account") + .getAsJsonObject("admin_account") + .get("account_id") + .isJsonPrimitive() + && + accountInfo.getAsJsonObject("account") + .getAsJsonObject("admin_account") + .getAsJsonPrimitive("account_id") + .getAsString() + .equals(account.toString()); + + if (createAdminIfDevAndNoAdminsExist(backendUri, token, account)) { + isAdmin = true; + } + + return Optional.of(new AccountInfo(account, name, isAdmin, openIdInfo.data())); + } + + return Optional.empty(); + } + + @Override + public AccountInfo account(OpenIdInfo openIdInfo, OpenidProvider provider) throws IOException, InterruptedException { + Optional maybeExistingAccount = getAccountIfExists(openIdInfo, provider); + + if (maybeExistingAccount.isPresent()) { + return maybeExistingAccount.get(); + } + // The login credentials do no exist yet, create a new account and return it. + else { + // create account + JwtCreator jwtCreator = new JwtCreator(Config.jwtSigningSecret()); + String token = jwtCreator.createAdminJwt(); + URI createAccountEndpoint = URI.create(backendUri + "/account"); + String newAccountJson = postJsonAsAdmin(createAccountEndpoint, "{}", token); + String newAccountId = JsonParser.parseString(newAccountJson) + .getAsJsonArray() + .get(0) + .getAsJsonObject() + .getAsJsonPrimitive("id") + .getAsString(); + + UUID accountId = UUID.fromString(newAccountId); + createLoginForAccount(accountId, openIdInfo, provider, backendUri, token); + + boolean isAdmin = createAdminIfDevAndNoAdminsExist(backendUri, token, accountId); + + return new AccountInfo(accountId, openIdInfo.name(), isAdmin, openIdInfo.data()); + } + } + + public AccountInfo useInviteToCreateAccount(UUID inviteId, OpenIdInfo openIdInfo, OpenidProvider provider) throws IOException, InterruptedException, RsdAccountInviteException { + Objects.requireNonNull(inviteId); + URI accountInviteUrl = URI.create(backendUri + "/account_invite?select=uses_left,expires_at&id=eq.%s".formatted(inviteId)); + JwtCreator jwtCreator = new JwtCreator(Config.jwtSigningSecret()); + String token = jwtCreator.createAdminJwt(); + + String response = getAsAdmin(accountInviteUrl, token); + + Integer usesLeft = checkInviteResponseGetUsesLeft(inviteId, response); + + // invite is valid + // decrease uses if finite + if (usesLeft != null) { + int usesDecreased = usesLeft - 1; + String patchInviteJson = "{\"uses_left\": %d}".formatted(usesDecreased); + URI accountInvitePatchUrl = URI.create(backendUri + "/account_invite?id=eq." + inviteId); + patchJsonAsAdmin(accountInvitePatchUrl, patchInviteJson, token); + } + + return account(openIdInfo, provider); + } + + static Integer checkInviteResponseGetUsesLeft(UUID inviteId, String json) throws RsdAccountInviteException { + // check if invite exists and is valid + JsonArray jsonArray = JsonParser.parseString(json).getAsJsonArray(); + if (jsonArray.isEmpty()) { + throw new RsdAccountInviteException("No invite with ID %s found".formatted(inviteId)); + } + + JsonObject inviteObject = jsonArray.get(0).getAsJsonObject(); + + ZonedDateTime expiresAt = ZonedDateTime.parse(inviteObject.getAsJsonPrimitive("expires_at").getAsString()); + if (expiresAt.isBefore(ZonedDateTime.now())) { + throw new RsdAccountInviteException("The invite with ID %s is expired".formatted(inviteId)); + } + + JsonElement usesLeft = inviteObject.get("uses_left"); + if (usesLeft.isJsonPrimitive() && usesLeft.getAsInt() <= 0) { + throw new RsdAccountInviteException("The invite with ID %s has been used up".formatted(inviteId)); + } + + return usesLeft.isJsonPrimitive() ? usesLeft.getAsInt() : null; + } + + public boolean isAdmin(UUID accountId) throws IOException, InterruptedException { + Objects.requireNonNull(accountId); + URI accountUrl = URI.create(backendUri + "/admin_account?account_id=eq.%s".formatted(accountId)); + JwtCreator jwtCreator = new JwtCreator(Config.jwtSigningSecret()); + String token = jwtCreator.createAdminJwt(); + + String response = getAsAdmin(accountUrl, token); + return parseIsAdminResponse(accountId, response); + } + + static boolean parseIsAdminResponse(UUID accountId, String response) { + Objects.requireNonNull(accountId); + Objects.requireNonNull(response); + JsonElement jsonTree = JsonParser.parseString(response); + return jsonTree.isJsonArray() + && jsonTree.getAsJsonArray().size() == 1 + && jsonTree.getAsJsonArray().get(0).getAsJsonObject().get("account_id").isJsonPrimitive() + && jsonTree.getAsJsonArray() + .get(0) + .getAsJsonObject() + .getAsJsonPrimitive("account_id") + .getAsString() + .equals(accountId.toString()); + } + + public void coupleLogin(UUID accountId, OpenIdInfo openIdInfo, OpenidProvider provider) throws IOException, InterruptedException { + JwtCreator jwtCreator = new JwtCreator(Config.jwtSigningSecret()); + String adminJwt = jwtCreator.createAdminJwt(); + createLoginForAccount(accountId, openIdInfo, provider, backendUri, adminJwt); + } + + private boolean createAdminIfDevAndNoAdminsExist(String backendUri, String token, UUID accountId) throws IOException, InterruptedException { + if (!Config.isDevEnv()) { + return false; + } + + URI adminAccountUri = URI.create(backendUri + "/admin_account"); + HttpResponse countAdminResponse = headAsAdmin(adminAccountUri, token, PREFER_HEADER_KEY, "count=exact"); + + String contentRange = countAdminResponse.headers().firstValue("Content-Range").orElseThrow(); + if (!"0".equals(contentRange.split("/")[1])) { + return false; + } + + String body = "{\"account_id\": \"%s\"}".formatted(accountId); + postJsonAsAdmin(adminAccountUri, body, token); + + return true; + } + + private void createLoginForAccount(UUID accountId, OpenIdInfo openIdInfo, OpenidProvider provider, String backendUri, String adminJwt) throws IOException, InterruptedException { + JsonObject loginForAccountData = new JsonObject(); + loginForAccountData.addProperty("account", accountId.toString()); + loginForAccountData.addProperty("sub", openIdInfo.sub()); + loginForAccountData.addProperty("name", openIdInfo.name()); + loginForAccountData.addProperty("email", openIdInfo.email()); + loginForAccountData.addProperty("home_organisation", openIdInfo.organisation()); + loginForAccountData.addProperty("provider", provider.toString()); + loginForAccountData.addProperty("last_login_date", ZonedDateTime.now() + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + URI createLoginUri = URI.create(backendUri + "/login_for_account"); + + HttpResponse response = postJsonAsAdminWithResponse(createLoginUri, loginForAccountData.toString(), adminJwt); + if (response.statusCode() == 409) { + throw new RsdAuthenticationException("This login is already coupled to an account."); + } else if (response.statusCode() >= 300) { + throw new RuntimeException("Error fetching data from the endpoint: %s with status code %d and response: %s".formatted(createLoginUri.toString(), response.statusCode(), response.body())); + } + } + + private void updateLoginForAccount(UUID id, OpenIdInfo openIdInfo, String authToken) throws IOException, InterruptedException { + JsonObject loginForAccountData = new JsonObject(); + loginForAccountData.addProperty("name", openIdInfo.name()); + loginForAccountData.addProperty("email", openIdInfo.email()); + loginForAccountData.addProperty("home_organisation", openIdInfo.organisation()); + loginForAccountData.addProperty("last_login_date", ZonedDateTime.now() + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + String resultingJson = loginForAccountData.toString(); + + URI patchLoginForAccountUri = URI.create(backendUri + "/login_for_account?id=eq." + id.toString()); + + patchJsonAsAdmin(patchLoginForAccountUri, resultingJson, authToken); + } + + static String getAsAdmin(URI uri, String token) throws IOException, InterruptedException { + HttpRequest request = HttpRequest.newBuilder() + .GET() + .uri(uri) + .header(AUTHORIZATION_HEADER_KEY, BEARER_HEADER_VALUE_PREFIX + " " + token) + .build(); + try (HttpClient client = HttpClient.newHttpClient()) { + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() >= 300) { + throw new RuntimeException("Error fetching data from the endpoint: " + uri.toString() + " with response: " + response.body()); + } + return response.body(); + } + } + + public static String postJsonAsAdmin(URI uri, String json, String token, String... headers) throws IOException, InterruptedException { + HttpResponse response = postJsonAsAdminWithResponse(uri, json, token, headers); + if (response.statusCode() >= 300) { + throw new RuntimeException("Error fetching data from the endpoint: %s with status code %d and response: %s".formatted(uri.toString(), response.statusCode(), response.body())); + } + return response.body(); + } + + public static HttpResponse postJsonAsAdminWithResponse(URI uri, String json, String token, String... headers) throws IOException, InterruptedException { + HttpRequest.Builder httpRequestBuilder = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(json)) + .uri(uri) + .header("Content-Type", "application/json") + .header(PREFER_HEADER_KEY, "return=representation") + .header(AUTHORIZATION_HEADER_KEY, BEARER_HEADER_VALUE_PREFIX + " " + token); + if (headers != null && headers.length > 0 && headers.length % 2 == 0) { + httpRequestBuilder.headers(headers); + } + HttpRequest request = httpRequestBuilder.build(); + try (HttpClient client = HttpClient.newHttpClient()) { + return client.send(request, HttpResponse.BodyHandlers.ofString()); + } + } + + private String patchJsonAsAdmin(URI uri, String json, String authToken) throws IOException, InterruptedException { + HttpRequest request = HttpRequest.newBuilder() + .method("PATCH", HttpRequest.BodyPublishers.ofString(json)) + .uri(uri) + .header("Content-Type", "application/json") + .header(PREFER_HEADER_KEY, "return=representation") + .header(AUTHORIZATION_HEADER_KEY, BEARER_HEADER_VALUE_PREFIX + " " + authToken) + .build(); + HttpResponse response; + try (HttpClient client = HttpClient.newHttpClient()) { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() >= 300) { + throw new RuntimeException("Error fetching data from the endpoint: " + uri.toString() + " with response: " + response.body()); + } + return response.body(); + } + } + + private HttpResponse headAsAdmin(URI uri, String token, String... headers) throws IOException, InterruptedException { + HttpRequest.Builder httpRequestBuilder = HttpRequest.newBuilder() + .HEAD() + .uri(uri) + .header(AUTHORIZATION_HEADER_KEY, BEARER_HEADER_VALUE_PREFIX + " " + token); + if (headers != null && headers.length > 0 && headers.length % 2 == 0) { + httpRequestBuilder.headers(headers); + } + HttpRequest request = httpRequestBuilder.build(); + try (HttpClient client = HttpClient.newHttpClient()) { + return client.send(request, HttpResponse.BodyHandlers.discarding()); + } + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/QueryParameterBuilder.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/QueryParameterBuilder.java new file mode 100644 index 0000000..18211e4 --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/QueryParameterBuilder.java @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.StringJoiner; + +/** + * This class builds a query parameter string, consisting of multiple key-value pairs of the form key=value, + * separated by the & character. + * Keys and values are both URL escaped. + * The resulting string is prepended with the ? character. + */ +public class QueryParameterBuilder { + + private final StringJoiner queryBuilder = new StringJoiner("&", "?", ""); + + /** + * @param key The key of the query parameter, as in key=value. + * @param value the value of the query parameter, as in key=value. + * @return This {@link QueryParameterBuilder}, so it can be used as in the builder pattern. + */ + public QueryParameterBuilder addQueryParameter(String key, String value) { + String keyEscaped = URLEncoder.encode(key, StandardCharsets.UTF_8); + String valueEscaped = URLEncoder.encode(value, StandardCharsets.UTF_8); + queryBuilder.add(keyEscaped + "=" + valueEscaped); + + return this; + } + + @Override + public String toString() { + return queryBuilder.toString(); + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdAccountInviteException.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdAccountInviteException.java new file mode 100644 index 0000000..4f70526 --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdAccountInviteException.java @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +public class RsdAccountInviteException extends Exception { + public RsdAccountInviteException(String message) { + super(message); + } + + public RsdAccountInviteException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdAuthenticationException.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdAuthenticationException.java new file mode 100644 index 0000000..19dc29a --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdAuthenticationException.java @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +public class RsdAuthenticationException extends RuntimeException { + + public RsdAuthenticationException(String message) { + super(message); + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdProviderData.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdProviderData.java new file mode 100644 index 0000000..1e41088 --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdProviderData.java @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import java.net.URI; + +public record RsdProviderData( + OpenidProvider openidProvider, + OpenidProviderAccessMethod accessType, + URI wellKnownUrl, + String displayName, + String htmlDescription, + int order +) { +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdProviders.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdProviders.java new file mode 100644 index 0000000..ff725cd --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdProviders.java @@ -0,0 +1,347 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Comparator; +import java.util.EnumMap; +import java.util.Map; +import java.util.SequencedCollection; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * This class exposes all active OpenID providers that users can sign in with. + * It does so by reading the relevant env variables and by requesting the 'authorization' URLs from the respective well-known URLs. + *
+ *
+ * The main function through which the data can be obtained is {@link #activeProvidersAsJson() activeProvidersAsJson} + *
+ *
+ * The data is obtained asynchronously in the background, meaning that subsequent calls to {@link #activeProvidersAsJson() activeProvidersAsJson} might return different data. + */ +public class RsdProviders { + + private static final Logger LOGGER = LoggerFactory.getLogger(RsdProviders.class); + + private static final OpenidProviderAccessMethodOrder DEFAULT_ACCESS_METHOD_ORDER = new OpenidProviderAccessMethodOrder(OpenidProviderAccessMethod.DISABLED, Integer.MAX_VALUE); + private final ConcurrentMap accessMethodOrderMap = new ConcurrentHashMap<>(); + private String activeProvidersString = "[]"; + private final ReadWriteLock jsonProvidersLock = new ReentrantReadWriteLock(false); + + public RsdProviders() { + Map rawAccessMethodMap = RsdProviders.parseAuthProvidersEnvString(System.getenv("RSD_AUTH_PROVIDERS")); + logRawAccessMap(rawAccessMethodMap); + SequencedCollection activeProvidersSorted = new TreeSet<>(Comparator.comparingInt(RsdProviderData::order)); + + for (OpenidProvider openidProvider : OpenidProvider.values()) { + OpenidProviderAccessMethodOrder rawAccessMethodOrder = rawAccessMethodMap.getOrDefault(openidProvider, DEFAULT_ACCESS_METHOD_ORDER); + OpenidProviderAccessMethod rawAccessMethod = rawAccessMethodOrder.accessMethod(); + if (!rawAccessMethod.isActive()) { + continue; + } + + String displayName = obtainDisplayName(openidProvider); + String htmlDescription = obtainHtmlDescription(openidProvider); + URI wellKnownUrl; + try { + wellKnownUrl = Config.wellKnownUrl(openidProvider); + if (openidProvider != OpenidProvider.local && wellKnownUrl == null) { + accessMethodOrderMap.put(openidProvider, OpenidProviderAccessMethod.MISCONFIGURED); + LOGGER.warn("Provider {} is misconfigured because no well-known URL was found", openidProvider); + continue; + } + } catch (URISyntaxException ignored) { + accessMethodOrderMap.put(openidProvider, OpenidProviderAccessMethod.MISCONFIGURED); + LOGGER.warn("Provider {} is misconfigured because it has an invalid well-known URL", openidProvider); + continue; + } + + activeProvidersSorted.add(new RsdProviderData(openidProvider, rawAccessMethod, wellKnownUrl, displayName, htmlDescription, rawAccessMethodOrder.order())); + } + + Map signInUrls = new EnumMap<>(OpenidProvider.class); + ExecutorService threadPool = Executors.newFixedThreadPool(10); + for (RsdProviderData rsdProviderData : activeProvidersSorted) { + threadPool.submit(() -> { + OpenidProvider openidProvider = rsdProviderData.openidProvider(); + try { + URI signInUrl = obtainSignInUrl(openidProvider, rsdProviderData.wellKnownUrl()); + accessMethodOrderMap.put(openidProvider, rsdProviderData.accessType()); + + Lock writeLock = jsonProvidersLock.writeLock(); + writeLock.lock(); + try { + signInUrls.put(openidProvider, signInUrl); + this.activeProvidersString = activeProvidersSorted + .stream() + .filter(p -> signInUrls.containsKey(p.openidProvider())) + .map(p -> toJson(p, signInUrls.get(p.openidProvider()))) + .collect(JsonArray::new, JsonArray::add, JsonArray::addAll) + .toString(); + } finally { + writeLock.unlock(); + } + } catch (IOException e) { + LOGGER.warn("IOException when getting sign in URL for provider {}", openidProvider, e); + } catch (RsdResponseException e) { + LOGGER.warn("RsdResponseException when getting sign in URL for provider {}", openidProvider, e); + } catch (InterruptedException e) { + LOGGER.warn("InterruptedException when getting sign in URL for provider {}", openidProvider, e); + Thread.currentThread().interrupt(); + } + }); + } + + threadPool.shutdown(); + } + + private static JsonObject toJson(RsdProviderData data, URI signInUrl) { + JsonObject jsonObject = new JsonObject(); + + jsonObject.addProperty("openidProvider", data.openidProvider().toString()); + jsonObject.addProperty("accessType", data.accessType().toString()); + jsonObject.addProperty("name", data.displayName()); + jsonObject.addProperty("html", data.htmlDescription()); + jsonObject.addProperty("signInUrl", signInUrl.toString()); + + return jsonObject; + } + + private static void logRawAccessMap(Map rawAccessMethodMap) { + for (OpenidProvider openidProvider : OpenidProvider.values()) { + OpenidProviderAccessMethod accessMethod = rawAccessMethodMap.getOrDefault(openidProvider, DEFAULT_ACCESS_METHOD_ORDER) + .accessMethod(); + + switch (accessMethod) { + case MISCONFIGURED -> + LOGGER.warn("Provider {} is misconfigured", openidProvider.toUserFriendlyString()); + case DISABLED -> LOGGER.info("Provider {} is disabled", openidProvider.toUserFriendlyString()); + case INVITE_ONLY -> + LOGGER.info("Provider {} is enabled with invites only", openidProvider.toUserFriendlyString()); + case EVERYONE -> { + if (openidProvider == OpenidProvider.local) { + LOGGER.warn("********************"); + LOGGER.warn("Warning: local accounts are enabled, this is not safe for production!"); + LOGGER.warn("********************"); + } else { + LOGGER.info("Provider {} is enabled for everyone", openidProvider.toUserFriendlyString()); + } + } + } + } + } + + record OpenidProviderAccessMethodOrder(OpenidProviderAccessMethod accessMethod, int order) { + } + + /** + * @param authConf A string containing a config for {@link OpenidProvider OpenID} providers, as described in .env.example + * for the RSD_AUTH_PROVIDERS variable. + * @return A map, that provides for each provider in authConf the parsed {@link OpenidProviderAccessMethod} + * and the index at which the provider was found, which can used for sorting on the input order. + * Missing entries are considered to be DISABLED; + * entries appearing multiple times are considered to be MISCONFIGURED. + */ + static Map parseAuthProvidersEnvString(String authConf) { + Map result = new EnumMap<>(OpenidProvider.class); + + if (authConf == null) { + return result; + } + String[] split = authConf.split(";"); + for (int i = 0; i < split.length; ++i) { + String providerConf = split[i]; + String[] providerSplit = providerConf.split(":"); + if (providerSplit.length == 0) { + continue; + } + OpenidProvider openidProvider; + try { + openidProvider = OpenidProvider.valueOf(providerSplit[0].toLowerCase()); + } catch (IllegalArgumentException e) { + continue; + } + + if (providerSplit.length != 2) { + result.put(openidProvider, new OpenidProviderAccessMethodOrder(OpenidProviderAccessMethod.MISCONFIGURED, i)); + continue; + } + + OpenidProviderAccessMethod accessMethod; + try { + accessMethod = OpenidProviderAccessMethod.valueOf(providerSplit[1]); + } catch (IllegalArgumentException e) { + result.put(openidProvider, new OpenidProviderAccessMethodOrder(OpenidProviderAccessMethod.MISCONFIGURED, i)); + continue; + } + + final int iCopy = i; + // duplicate entries are considered to be misconfigured + result.compute(openidProvider, (_key, oldValue) -> oldValue == null ? new OpenidProviderAccessMethodOrder(accessMethod, iCopy) : + new OpenidProviderAccessMethodOrder(OpenidProviderAccessMethod.MISCONFIGURED, iCopy)); + } + + return result; + } + + /** + * @param provider The provider to get the access method of. + * @return The access method of the OpenID provider, incorporating both the env variable + * RSD_AUTH_PROVIDERS, the respective well-known URL env variable and + * the result of trying to obtain the authorize endpoint from the + * well-known URL. + *
+ *
+ * Calling this multiple times with the same parameter might yield different results, + * as the data from the well-known URL is fetched in the background. + */ + public OpenidProviderAccessMethod accessMethodOfProvider(OpenidProvider provider) { + return accessMethodOrderMap.getOrDefault(provider, OpenidProviderAccessMethod.DISABLED); + } + + /** + * @return A string representing a JSON array, containing the active OpenID providers. + *
+ *
+ * Calling this multiple times might yield different results, + * as the data from the well-known URL is fetched in the background. + */ + public String activeProvidersAsJson() { + Lock readLock = jsonProvidersLock.readLock(); + readLock.lock(); + try { + return activeProvidersString; + } finally { + readLock.unlock(); + } + } + + private static String obtainDisplayName(OpenidProvider provider) { + return switch (provider) { + case local -> "local account"; + case surfconext -> "SURFconext"; + case helmholtz -> "Helmholtz ID"; + case orcid -> "ORCID"; + case azure -> Utils.coalesce(System.getenv("AZURE_DISPLAY_NAME"), "Azure Active Directory"); + case linkedin -> "LinkedIn"; + }; + } + + private static String obtainHtmlDescription(OpenidProvider provider) { + return switch (provider) { + case local -> + "Sign in with local account is for testing purposes only. This option should not be enabled in the production version."; + case surfconext -> + "Sign in with SURFconext is for Dutch Institutions who enabled the RSD service in the SURFconext IdP dashboard."; + case helmholtz -> + "Sign in with Helmholtz ID is enabled for all members of the Helmholtz Research Foundation."; + case orcid -> "Sign in with your ORCID account."; + case azure -> + Utils.coalesce(System.getenv("AZURE_DESCRIPTION_HTML"), "Login with your institutional credentials."); + case linkedin -> "Sign in with your LinkedIn account."; + }; + } + + private static URI obtainSignInUrl(OpenidProvider provider, URI wellKnownUrl) throws IOException, InterruptedException, RsdResponseException { + return switch (provider) { + case local -> URI.create("/login/local"); + case surfconext, helmholtz, orcid, azure, linkedin -> { + URI authUrl = Utils.getAuthUrlFromWellKnownUrl(wellKnownUrl); + yield addQueryParametersToAuthUrl(authUrl, provider); + } + }; + } + + private static String obtainRedirectUrl(OpenidProvider provider) { + return switch (provider) { + case local -> null; + case surfconext -> System.getenv("SURFCONEXT_REDIRECT"); + case helmholtz -> System.getenv("HELMHOLTZID_REDIRECT"); + case orcid -> System.getenv("ORCID_REDIRECT"); + case azure -> System.getenv("AZURE_REDIRECT"); + case linkedin -> System.getenv("LINKEDIN_REDIRECT"); + }; + } + + private static String obtainClientId(OpenidProvider provider) { + return switch (provider) { + case local -> null; + case surfconext -> System.getenv("SURFCONEXT_CLIENT_ID"); + case helmholtz -> System.getenv("HELMHOLTZID_CLIENT_ID"); + case orcid -> System.getenv("ORCID_CLIENT_ID"); + case azure -> System.getenv("AZURE_CLIENT_ID"); + case linkedin -> System.getenv("LINKEDIN_CLIENT_ID"); + }; + } + + private static URI addQueryParametersToAuthUrl(URI authUrl, OpenidProvider openidProvider) { + String queryParameters = switch (openidProvider) { + case local -> ""; + case surfconext -> { + QueryParameterBuilder queryParameterBuilder = new QueryParameterBuilder() + .addQueryParameter("response_mode", "form_post") + .addQueryParameter("response_type", "code") + .addQueryParameter("scope", "openid") + .addQueryParameter("redirect_uri", obtainRedirectUrl(openidProvider)) + .addQueryParameter("client_id", obtainClientId(openidProvider)) + .addQueryParameter("claims", "{\"id_token\":{\"schac_home_organization\":null,\"name\":null,\"email\":null}}"); + + if (Config.isDevEnv()) { + queryParameterBuilder.addQueryParameter("prompt", "login"); + } + + yield queryParameterBuilder.toString(); + } + case helmholtz -> new QueryParameterBuilder() + .addQueryParameter("response_mode", "query") + .addQueryParameter("response_type", "code") + .addQueryParameter("scope", "openid profile email eduperson_principal_name") + .addQueryParameter("redirect_uri", obtainRedirectUrl(openidProvider)) + .addQueryParameter("client_id", obtainClientId(openidProvider)) + .addQueryParameter("claims", "{\"id_token\":{\"schac_home_organization\":null,\"name\":null,\"email\":null}}") + .toString(); + case orcid -> new QueryParameterBuilder() + .addQueryParameter("response_mode", "query") + .addQueryParameter("response_type", "code") + .addQueryParameter("scope", "openid") + .addQueryParameter("redirect_uri", obtainRedirectUrl(openidProvider)) + .addQueryParameter("client_id", obtainClientId(openidProvider)) + .toString(); + case azure -> new QueryParameterBuilder() + .addQueryParameter("response_mode", "query") + .addQueryParameter("response_type", "code") + .addQueryParameter("scope", "openid") + .addQueryParameter("prompt", "select_account") + .addQueryParameter("redirect_uri", obtainRedirectUrl(openidProvider)) + .addQueryParameter("client_id", obtainClientId(openidProvider)) + .toString(); + case linkedin -> new QueryParameterBuilder() + .addQueryParameter("response_mode", "query") + .addQueryParameter("response_type", "code") + .addQueryParameter("scope", "openid profile email") + .addQueryParameter("redirect_uri", obtainRedirectUrl(openidProvider)) + .addQueryParameter("client_id", obtainClientId(openidProvider)) + .toString(); + }; + + String urlAsString = authUrl.toString(); + return URI.create(urlAsString + queryParameters); + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdResponseException.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdResponseException.java new file mode 100644 index 0000000..48e42c3 --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/RsdResponseException.java @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2022 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import java.net.URI; + +public class RsdResponseException extends Exception { + + private static final long serialVersionUID = 1L; + + public final int statusCode; + public final URI uri; + public final String body; + + public RsdResponseException(int statusCode, URI uri, String body, String message) { + super(message); + this.statusCode = statusCode; + this.uri = uri; + this.body = body; + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/SurfconextLogin.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/SurfconextLogin.java new file mode 100644 index 0000000..8f5c6ef --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/SurfconextLogin.java @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: 2021 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2021 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +// SPDX-FileCopyrightText: 2022 Matthias Rüster (GFZ) +// SPDX-FileCopyrightText: 2022 dv4all +// SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.interfaces.DecodedJWT; +import com.google.gson.JsonParser; + +import java.io.IOException; +import java.net.URI; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class SurfconextLogin implements Login { + + private final String code; + private final String redirectUrl; + + public SurfconextLogin(String code, String redirectUrl) { + this.code = Objects.requireNonNull(code); + this.redirectUrl = Objects.requireNonNull(redirectUrl); + } + + @Override + public OpenIdInfo openidInfo() throws IOException, InterruptedException, RsdResponseException { + Map form = createForm(); + String tokenResponse = getTokensFromSurfconext(form); + String idToken = extractIdToken(tokenResponse); + + DecodedJWT idJwt = JWT.decode(idToken); + String subject = idJwt.getSubject(); + String name = idJwt.getClaim("name").asString(); + String email = idJwt.getClaim("email").asString(); + String organisation = idJwt.getClaim("schac_home_organization").asString(); + Map> emptyData = Collections.emptyMap(); + return new OpenIdInfo(subject, name, email, organisation, emptyData); + } + + private Map createForm() { + Map form = new HashMap<>(); + form.put("code", code); + form.put("grant_type", "authorization_code"); + form.put("redirect_uri", redirectUrl); + form.put("scope", "openid"); + form.put("client_id", Config.surfconextClientId()); + form.put("client_secret", Config.surfconextClientSecret()); + return form; + } + + private String getTokensFromSurfconext(Map form) throws IOException, InterruptedException, RsdResponseException { + URI tokenEndpoint = Utils.getTokenUrlFromWellKnownUrl(URI.create(Config.surfconextWellknown())); + return Utils.postForm(tokenEndpoint, form); + } + + private String extractIdToken(String response) { + return JsonParser.parseString(response).getAsJsonObject().getAsJsonPrimitive("id_token").getAsString(); + } +} diff --git a/authentication/src/main/java/nl/esciencecenter/rsd/authentication/Utils.java b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/Utils.java new file mode 100644 index 0000000..20940e9 --- /dev/null +++ b/authentication/src/main/java/nl/esciencecenter/rsd/authentication/Utils.java @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: 2022 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.StringJoiner; + +public class Utils { + + private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class); + + private Utils() { + } + + @SafeVarargs + public static T coalesce(T... objects) { + for (T object : objects) { + if (object != null) { + return object; + } + } + throw new NullPointerException("No non-null reference found"); + } + + public static URI getAuthUrlFromWellKnownUrl(URI wellKnownUrl) throws IOException, InterruptedException, RsdResponseException { + HttpRequest request = HttpRequest.newBuilder(wellKnownUrl).build(); + try (HttpClient client = HttpClient.newHttpClient()) { + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + + if (response.statusCode() != 200) { + throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Wrong status code querying well-known URL"); + } + try { + return extractAuthUrlFromWellKnownData(response.body()); + } catch (RuntimeException e) { + throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Could not extract auth URL from well-known URL response body"); + } + } + } + + public static URI getTokenUrlFromWellKnownUrl(URI wellKnownUrl) throws IOException, InterruptedException { + HttpRequest request = HttpRequest.newBuilder(wellKnownUrl).build(); + try (HttpClient client = HttpClient.newHttpClient()) { + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + return extractTokenUrlFromWellKnownData(response.body()); + } + } + + public static URI extractAuthUrlFromWellKnownData(String jsonData) { + JsonObject dataAsObject = JsonParser.parseString(jsonData).getAsJsonObject(); + String tokenUrl = dataAsObject.getAsJsonPrimitive("authorization_endpoint").getAsString(); + return URI.create(tokenUrl); + } + + public static URI extractTokenUrlFromWellKnownData(String jsonData) { + JsonObject dataAsObject = JsonParser.parseString(jsonData).getAsJsonObject(); + String tokenUrl = dataAsObject.getAsJsonPrimitive("token_endpoint").getAsString(); + return URI.create(tokenUrl); + } + + public static String urlEncode(String s) { + return URLEncoder.encode(s, StandardCharsets.UTF_8); + } + + public static String formMapToxWwwFormUrlencoded(Map form) { + StringJoiner x_www_form_urlencoded = new StringJoiner("&"); + form.forEach((key, value) -> x_www_form_urlencoded.add(urlEncode(key) + "=" + urlEncode(value))); + return x_www_form_urlencoded.toString(); + } + + public static HttpRequest formToHttpRequest(URI uri, Map form) { + String body = formMapToxWwwFormUrlencoded(form); + + return HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(body)) + .uri(uri) + .header("Content-Type", "application/x-www-form-urlencoded") + .build(); + } + + public static String postForm(URI uri, Map form) throws IOException, InterruptedException, RsdResponseException { + HttpRequest request = formToHttpRequest(uri, form); + try (HttpClient client = HttpClient.newHttpClient()) { + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() >= 300) { + LOGGER.error("Error posting form: {}, {}", response.statusCode(), response.body()); + throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Error posting form"); + } + return response.body(); + } + } +} diff --git a/authentication/src/main/resources/logback.xml b/authentication/src/main/resources/logback.xml new file mode 100644 index 0000000..a94f683 --- /dev/null +++ b/authentication/src/main/resources/logback.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} -%kvp- %msg%n + + + + + + + diff --git a/authentication/src/test/java/nl/esciencecenter/rsd/authentication/HelmholtzIdLoginTest.java b/authentication/src/test/java/nl/esciencecenter/rsd/authentication/HelmholtzIdLoginTest.java new file mode 100644 index 0000000..a7016a2 --- /dev/null +++ b/authentication/src/test/java/nl/esciencecenter/rsd/authentication/HelmholtzIdLoginTest.java @@ -0,0 +1,141 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2022 Matthias Rüster (GFZ) +// SPDX-FileCopyrightText: 2023 - 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2023 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import net.minidev.json.JSONArray; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class HelmholtzIdLoginTest { + @Test + void testOrganisationFromEntitlementsExpected() { + JSONArray arr = new JSONArray(); + + arr.add("urn:geant:h-df.de:group:test-vo#login-dev.helmholtz.de"); + assertNull( + HelmholtzIdLogin.getOrganisationFromEntitlements(arr) + ); + + arr.add("urn:geant:helmholtz.de:group:Helmholtz-member#login-dev.helmholtz.de"); + arr.add("urn:geant:helmholtz.de:group:GFZ#login-dev.helmholtz.de"); + arr.add("urn:geant:helmholtz.de:group:HIFIS#login-dev.helmholtz.de"); + arr.add("urn:mace:dir:entitlement:common-lib-terms"); + + assertEquals( + "GFZ", + HelmholtzIdLogin.getOrganisationFromEntitlements(arr) + ); + + arr.add("urn:geant:helmholtz.de:group:UFZ#login-dev.helmholtz.de"); + assertTrue( + "GFZ".equals(HelmholtzIdLogin.getOrganisationFromEntitlements(arr)) + || + "UFZ".equals(HelmholtzIdLogin.getOrganisationFromEntitlements(arr)) + ); + + JSONArray nohash = new JSONArray(); + + nohash.add("urn:geant:helmholtz.de:group:Helmholtz-member"); + nohash.add("urn:geant:helmholtz.de:group:GFZ"); + + assertEquals( + "GFZ", + HelmholtzIdLogin.getOrganisationFromEntitlements(nohash) + ); + } + + @Test + void testOrganisationFromEntitlementsExternalUsers() { + JSONArray arr = new JSONArray(); + + arr.add("urn:geant:helmholtz.de:group:Helmholtz-member#login-dev.helmholtz.de"); + arr.add("urn:geant:helmholtz.de:group:GFZ#login-dev.helmholtz.de"); + arr.add("urn:mace:dir:entitlement:common-lib-terms"); + + assertEquals( + "GFZ", + HelmholtzIdLogin.getOrganisationFromEntitlements(arr) + ); + + assertNull( + HelmholtzIdLogin.getOrganisationFromEntitlements(null) + ); + + JSONArray empty = new JSONArray(); + + assertNull( + HelmholtzIdLogin.getOrganisationFromEntitlements(empty) + ); + } + + @Test + void testOrganisationFromEntitlementsRegex() { + JSONArray arr = new JSONArray(); + + arr.add("urn:geant:helmholtz.de:group:Helmholtz-member#login-dev.helmholtz.de"); + arr.add("urn:geant:helmholtzade:group:GFZ#login-dev.helmholtz.de"); + arr.add("urn@geant@helmholtz@de@group@GFZ#login-dev.helmholtz.de"); + arr.add("GFZ#login-dev.helmholtz.de"); + arr.add("GFZ"); + + assertEquals( + HelmholtzIdLogin.DEFAULT_ORGANISATION, + HelmholtzIdLogin.getOrganisationFromEntitlements(arr) + ); + } + + @Test + void testOrganisationFromEntitlementsDefaultValue() { + JSONArray arr = new JSONArray(); + + // No entitlements -> Null + assertNull( + HelmholtzIdLogin.getOrganisationFromEntitlements(null) + ); + + assertNull( + HelmholtzIdLogin.getOrganisationFromEntitlements(arr) + ); + + // Default value should be returned if user is part of the Helmholtz + // association but no fitting centre could be found + arr.add("urn:geant:helmholtz.de:group:Helmholtz-member#login.helmholtz.de"); + + assertEquals( + HelmholtzIdLogin.DEFAULT_ORGANISATION, + HelmholtzIdLogin.getOrganisationFromEntitlements(arr) + ); + + // Unknown or new institution + arr.add("urn:geant:helmholtz.de:group:NoRealCentre#login-dev.helmholtz.de"); + assertEquals( + HelmholtzIdLogin.DEFAULT_ORGANISATION, + HelmholtzIdLogin.getOrganisationFromEntitlements(arr) + ); + + arr.add(""); + arr.add("test"); + + assertEquals( + HelmholtzIdLogin.DEFAULT_ORGANISATION, + HelmholtzIdLogin.getOrganisationFromEntitlements(arr) + ); + + // Actual centre does not return default value + arr.add("urn:geant:helmholtz.de:group:GFZ#login-dev.helmholtz.de"); + assertNotEquals( + HelmholtzIdLogin.DEFAULT_ORGANISATION, + HelmholtzIdLogin.getOrganisationFromEntitlements(arr) + ); + } +} diff --git a/authentication/src/test/java/nl/esciencecenter/rsd/authentication/MainTest.java b/authentication/src/test/java/nl/esciencecenter/rsd/authentication/MainTest.java new file mode 100644 index 0000000..7b768ba --- /dev/null +++ b/authentication/src/test/java/nl/esciencecenter/rsd/authentication/MainTest.java @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: 2022 - 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2022 Matthias Rüster (GFZ) +// SPDX-FileCopyrightText: 2023 - 2025 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2023 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +class MainTest { + Map> emptyData = Collections.emptyMap(); + OpenIdInfo userinfo = new OpenIdInfo( + "12345", "User Name", "user@example.com", "Example User", emptyData + ); + OpenIdInfo userinfoNullOrganisation = new OpenIdInfo( + "12345", "User Name", "user@example.com", null, emptyData + ); + static MockedStatic utilities; + + @BeforeAll + static void init() { + utilities = Mockito.mockStatic(Config.class); + } + + @AfterAll + static void cleanup() { + utilities.close(); + } +} diff --git a/authentication/src/test/java/nl/esciencecenter/rsd/authentication/PostgrestAccountTest.java b/authentication/src/test/java/nl/esciencecenter/rsd/authentication/PostgrestAccountTest.java new file mode 100644 index 0000000..046d691 --- /dev/null +++ b/authentication/src/test/java/nl/esciencecenter/rsd/authentication/PostgrestAccountTest.java @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.time.ZonedDateTime; +import java.util.UUID; + +class PostgrestAccountTest { + + @Test + void givenEmtpyArray_whenCheckingIfAdmin_thenFalseReturned() { + String emptyArray = "[]"; + UUID adminUuid = UUID.randomUUID(); + + Assertions.assertFalse(PostgrestAccount.parseIsAdminResponse(adminUuid, emptyArray)); + } + + @Test + void givenResponseWithNullValue_whenCheckingIfAdmin_thenFalseReturned() { + String emptyArray = "[{\"account_id\": null}]"; + UUID adminUuid = UUID.randomUUID(); + + Assertions.assertFalse(PostgrestAccount.parseIsAdminResponse(adminUuid, emptyArray)); + } + + @Test + void givenArrayOfSizeOneWithCorrectUuid_whenCheckingIfAdmin_thenTrueReturned() { + UUID adminUuid = UUID.randomUUID(); + String successResponse = "[{\"account_id\": \"%s\"}]".formatted(adminUuid); + + Assertions.assertTrue(PostgrestAccount.parseIsAdminResponse(adminUuid, successResponse)); + } + + @Test + void givenArrayOfSizeOneWithIncorrectUuid_whenCheckingIfAdmin_thenFalseReturned() { + UUID adminUuid = UUID.randomUUID(); + String successResponse = "[{\"account_id\": \"%s\"}]".formatted(UUID.randomUUID()); + + Assertions.assertFalse(PostgrestAccount.parseIsAdminResponse(adminUuid, successResponse)); + } + + @Test + void givenArrayOfSizeTwoWithCorrectUuid_whenCheckingIfAdmin_thenFalseReturned() { + UUID adminUuid = UUID.randomUUID(); + String wrongIdResponse = "[{\"account_id\": \"%s\"}, {\"account_id\": \"%s\"}]".formatted(adminUuid, UUID.randomUUID()); + + Assertions.assertFalse(PostgrestAccount.parseIsAdminResponse(adminUuid, wrongIdResponse)); + } + + private String formatInviteResponse(Integer usesLeft, ZonedDateTime expiresAt) { + return "[{\"uses_left\": %s, \"expires_at\": \"%s\"}]".formatted(usesLeft, expiresAt); + } + + @Test + void givenEmptyInviteResponse_whenParsed_thenExceptionThrown() { + String json = "[]"; + + Assertions.assertThrowsExactly(RsdAccountInviteException.class, () -> PostgrestAccount.checkInviteResponseGetUsesLeft(UUID.randomUUID(), json)); + } + + @Test + void givenValidInviteResponseWithFiniteUses_whenParsed_thenUsesLeftReturned() { + int expectedUsesLeft = 1; + String json = formatInviteResponse(expectedUsesLeft, ZonedDateTime.now().plusMinutes(1)); + + int actualUsesLeft = Assertions.assertDoesNotThrow(() -> PostgrestAccount.checkInviteResponseGetUsesLeft(UUID.randomUUID(), json)); + Assertions.assertEquals(expectedUsesLeft, actualUsesLeft); + } + + @Test + void givenValidInviteResponseWithInfiniteUses_whenParsed_thenNullReturned() { + String json = formatInviteResponse(null, ZonedDateTime.now().plusMinutes(1)); + + Integer actualUsesLeft = Assertions.assertDoesNotThrow(() -> PostgrestAccount.checkInviteResponseGetUsesLeft(UUID.randomUUID(), json)); + Assertions.assertNull(actualUsesLeft); + } + + @Test + void givenInviteResponseWithZeroUses_whenParsed_thenExceptionThrown() { + String json = formatInviteResponse(0, ZonedDateTime.now().plusMinutes(1)); + + Assertions.assertThrowsExactly(RsdAccountInviteException.class, () -> PostgrestAccount.checkInviteResponseGetUsesLeft(UUID.randomUUID(), json)); + } + + @Test + void givenInviteResponseWithNegativeUses_whenParsed_thenExceptionThrown() { + String json = formatInviteResponse(-1, ZonedDateTime.now().plusMinutes(1)); + + Assertions.assertThrowsExactly(RsdAccountInviteException.class, () -> PostgrestAccount.checkInviteResponseGetUsesLeft(UUID.randomUUID(), json)); + } + + @Test + void givenInviteResponseExpired_whenParsed_thenExceptionThrown() { + String json = formatInviteResponse(1, ZonedDateTime.now().minusMinutes(1)); + + Assertions.assertThrowsExactly(RsdAccountInviteException.class, () -> PostgrestAccount.checkInviteResponseGetUsesLeft(UUID.randomUUID(), json)); + } +} diff --git a/authentication/src/test/java/nl/esciencecenter/rsd/authentication/QueryParameterBuilderTest.java b/authentication/src/test/java/nl/esciencecenter/rsd/authentication/QueryParameterBuilderTest.java new file mode 100644 index 0000000..a13bd71 --- /dev/null +++ b/authentication/src/test/java/nl/esciencecenter/rsd/authentication/QueryParameterBuilderTest.java @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +class QueryParameterBuilderTest { + + @Test + void givenNoKeyValuePairs_whenCallingToString_thenQuestionMarkReturned() { + QueryParameterBuilder queryParameterBuilder = new QueryParameterBuilder(); + + Assertions.assertEquals("?", queryParameterBuilder.toString()); + } + + @ParameterizedTest + @CsvSource({ + "key,value,?key=value", + "123,456,?123=456", + "scope,openid profile email,?scope=openid+profile+email", + "url,https://www.example.com,?url=https%3A%2F%2Fwww.example.com", + "a?b&c,d&e?f=g,?a%3Fb%26c=d%26e%3Ff%3Dg", + }) + void givenKeyValuePairs_whenBuildingAndCallingToString_thenCorrectOutputProduced(String key, String value, String expected) { + QueryParameterBuilder queryParameterBuilder = new QueryParameterBuilder(); + + queryParameterBuilder.addQueryParameter(key, value); + + Assertions.assertEquals(expected, queryParameterBuilder.toString()); + } + + @Test + void givenMultipleKeyValuePairs_whenBuildingAndCallingToString_thenCorrectOutputProduced() { + QueryParameterBuilder queryParameterBuilder = new QueryParameterBuilder(); + + queryParameterBuilder.addQueryParameter("key", "value") + .addQueryParameter("123", "456") + .addQueryParameter("url", "https://www.example.com"); + + Assertions.assertEquals("?key=value&123=456&url=https%3A%2F%2Fwww.example.com", queryParameterBuilder.toString()); + } +} diff --git a/authentication/src/test/java/nl/esciencecenter/rsd/authentication/RsdProvidersTest.java b/authentication/src/test/java/nl/esciencecenter/rsd/authentication/RsdProvidersTest.java new file mode 100644 index 0000000..3631f47 --- /dev/null +++ b/authentication/src/test/java/nl/esciencecenter/rsd/authentication/RsdProvidersTest.java @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Map; + +class RsdProvidersTest { + + @Test + void whenGivenNullOrEmptyString_whenParsingAuthConfig_thenEmptyMapReturned() { + Map emptyMap = RsdProviders.parseAuthProvidersEnvString(null); + Assertions.assertTrue(emptyMap.isEmpty()); + + emptyMap = RsdProviders.parseAuthProvidersEnvString(""); + Assertions.assertTrue(emptyMap.isEmpty()); + } + + @Test + void whenGivenValidConfig_whenParsingAuthConfig_thenCorrectMapReturned() { + String authConfig = "SURFCONEXT:EVERYONE;HELMHOLTZ:INVITE_ONLY;ORCID:INVITE_ONLY;AZURE:EVERYONE"; + Map authMap = RsdProviders.parseAuthProvidersEnvString(authConfig); + + Assertions.assertEquals(4, authMap.size()); + Assertions.assertEquals(OpenidProviderAccessMethod.EVERYONE, authMap.get(OpenidProvider.surfconext) + .accessMethod()); + Assertions.assertEquals(OpenidProviderAccessMethod.EVERYONE, authMap.get(OpenidProvider.azure).accessMethod()); + Assertions.assertEquals(OpenidProviderAccessMethod.INVITE_ONLY, authMap.get(OpenidProvider.helmholtz) + .accessMethod()); + Assertions.assertEquals(OpenidProviderAccessMethod.INVITE_ONLY, authMap.get(OpenidProvider.orcid) + .accessMethod()); + + Assertions.assertTrue(authMap.get(OpenidProvider.surfconext).order() < authMap.get(OpenidProvider.helmholtz) + .order()); + Assertions.assertTrue(authMap.get(OpenidProvider.helmholtz).order() < authMap.get(OpenidProvider.orcid) + .order()); + Assertions.assertTrue(authMap.get(OpenidProvider.orcid).order() < authMap.get(OpenidProvider.azure).order()); + } + + @Test + void whenGivenConfigWithDuplicates_whenParsingAuthConfig_thenDuplicatesAreMisconfigured() { + String authConfig = "SURFCONEXT:EVERYONE;ORCID:INVITE_ONLY;SURFCONEXT:EVERYONE"; + Map authMap = RsdProviders.parseAuthProvidersEnvString(authConfig); + + Assertions.assertEquals(2, authMap.size()); + Assertions.assertEquals(OpenidProviderAccessMethod.MISCONFIGURED, authMap.get(OpenidProvider.surfconext) + .accessMethod()); + Assertions.assertEquals(OpenidProviderAccessMethod.INVITE_ONLY, authMap.get(OpenidProvider.orcid) + .accessMethod()); + } + + @Test + void whenGivenConfigWithNonExistingKey_whenParsingAuthConfig_thenThoseAreSkipped() { + String authConfig = "SURFCONEXT:EVERYONE;ORCID:INVITE_ONLY;DOESNOTEXIST:EVERYONE"; + Map authMap = Assertions.assertDoesNotThrow(() -> RsdProviders.parseAuthProvidersEnvString(authConfig)); + + Assertions.assertEquals(2, authMap.size()); + Assertions.assertEquals(OpenidProviderAccessMethod.EVERYONE, authMap.get(OpenidProvider.surfconext) + .accessMethod()); + Assertions.assertEquals(OpenidProviderAccessMethod.INVITE_ONLY, authMap.get(OpenidProvider.orcid) + .accessMethod()); + } + + @Test + void whenGivenConfigWithNonExistingValue_whenParsingAuthConfig_thenThoseAreMisconfigured() { + String authConfig = "SURFCONEXT:EVERYONE;ORCID:INVITE_ONLY;AZURE:MISTAKE;LOCAL:"; + Map authMap = RsdProviders.parseAuthProvidersEnvString(authConfig); + + Assertions.assertEquals(4, authMap.size()); + Assertions.assertEquals(OpenidProviderAccessMethod.EVERYONE, authMap.get(OpenidProvider.surfconext) + .accessMethod()); + Assertions.assertEquals(OpenidProviderAccessMethod.INVITE_ONLY, authMap.get(OpenidProvider.orcid) + .accessMethod()); + Assertions.assertEquals(OpenidProviderAccessMethod.MISCONFIGURED, authMap.get(OpenidProvider.azure) + .accessMethod()); + Assertions.assertEquals(OpenidProviderAccessMethod.MISCONFIGURED, authMap.get(OpenidProvider.local) + .accessMethod()); + + authConfig = "SURFCONEXT:EVERYONE:INVITE_ONLY;ORCID:INVITE_ONLY"; + authMap = RsdProviders.parseAuthProvidersEnvString(authConfig); + + Assertions.assertEquals(2, authMap.size()); + Assertions.assertEquals(OpenidProviderAccessMethod.MISCONFIGURED, authMap.get(OpenidProvider.surfconext) + .accessMethod()); + Assertions.assertEquals(OpenidProviderAccessMethod.INVITE_ONLY, authMap.get(OpenidProvider.orcid) + .accessMethod()); + } + + @Test + void whenGivenConfigWithMissingValue_whenParsingAuthConfig_thenThoseAreMisconfigured() { + String authConfig = "SURFCONEXT;ORCID:INVITE_ONLY;AZURE:EVERYONE"; + Map authMap = RsdProviders.parseAuthProvidersEnvString(authConfig); + + Assertions.assertEquals(3, authMap.size()); + Assertions.assertEquals(OpenidProviderAccessMethod.MISCONFIGURED, authMap.get(OpenidProvider.surfconext) + .accessMethod()); + Assertions.assertEquals(OpenidProviderAccessMethod.INVITE_ONLY, authMap.get(OpenidProvider.orcid) + .accessMethod()); + Assertions.assertEquals(OpenidProviderAccessMethod.EVERYONE, authMap.get(OpenidProvider.azure).accessMethod()); + } + + @ParameterizedTest + @ValueSource(strings = { + ";", + ";;;", + ":", + ":::", + ";:", + ":;", + ";;::;;:;;;;::", + "::;;;;:;;;:;;;;:;;:;:;:;", + "a=b;c=d:e=f", + }) + void whenGivenMisconfiguredConfig_whenParsingAuthConfig_thenNoExceptionThrown(String authConfig) { + Map authMap = Assertions.assertDoesNotThrow(() -> RsdProviders.parseAuthProvidersEnvString(authConfig)); + Assertions.assertTrue(authMap.isEmpty()); + } +} diff --git a/authentication/src/test/java/nl/esciencecenter/rsd/authentication/UtilsTest.java b/authentication/src/test/java/nl/esciencecenter/rsd/authentication/UtilsTest.java new file mode 100644 index 0000000..c196f77 --- /dev/null +++ b/authentication/src/test/java/nl/esciencecenter/rsd/authentication/UtilsTest.java @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.authentication; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.net.http.HttpHeaders; +import java.net.http.HttpRequest; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +class UtilsTest { + + @Test + void givenValidWellKnownData_whenExtractingTokenEndpoint_correctResultReturned() { + // editorconfig-checker-disable + String data = """ + { + "token_endpoint_auth_signing_alg_values_supported": [ + "RS256" + ], + "id_token_signing_alg_values_supported": [ + "RS256" + ], + "userinfo_endpoint": "https://sandbox.orcid.org/oauth/userinfo", + "authorization_endpoint": "https://sandbox.orcid.org/oauth/authorize", + "token_endpoint": "https://sandbox.orcid.org/oauth/token", + "jwks_uri": "https://sandbox.orcid.org/oauth/jwks", + "claims_supported": [ + "family_name", + "given_name", + "name", + "auth_time", + "iss", + "sub" + ], + "scopes_supported": [ + "openid" + ], + "subject_types_supported": [ + "public" + ], + "response_types_supported": [ + "code", + "id_token", + "id_token token" + ], + "claims_parameter_supported": false, + "token_endpoint_auth_methods_supported": [ + "client_secret_post" + ], + "grant_types_supported": [ + "authorization_code", + "implicit", + "refresh_token" + ], + "issuer": "https://sandbox.orcid.org" + }"""; + // editorconfig-checker-enable + + URI tokenEndpoint = Utils.extractTokenUrlFromWellKnownData(data); + + Assertions.assertEquals(URI.create("https://sandbox.orcid.org/oauth/token"), tokenEndpoint); + } + + @Test + void givenInvalidJson_whenExtractingTokenEndpoint_thenExceptionThrown() { + String data = "{"; + + Assertions.assertThrows(RuntimeException.class, () -> Utils.extractTokenUrlFromWellKnownData(data)); + } + + @Test + void givenDataWithoutTokenEndpoint_whenExtractingTokenEndpoint_thenExceptionThrown() { + String data = "{\"token_endpoint\": null}"; + + Assertions.assertThrows(ClassCastException.class, () -> Utils.extractTokenUrlFromWellKnownData(data)); + } + + @Test + void givenFormMap_whenConvertingToxWwwFormUrlencoded_thenCorrectResultReturned() { + Map emptyForm = Map.of(); + Assertions.assertEquals("", Utils.formMapToxWwwFormUrlencoded(emptyForm)); + + Map singleEntryForm = Map.of("abc", "def"); + Assertions.assertEquals("abc=def", Utils.formMapToxWwwFormUrlencoded(singleEntryForm)); + + // using a LinkedHashMap to ensure iteration order is insertion order + Map multiEntryForm = new LinkedHashMap<>(); + multiEntryForm.put("abc", "def"); + multiEntryForm.put("123", "456"); + Assertions.assertEquals("abc=def&123=456", Utils.formMapToxWwwFormUrlencoded(multiEntryForm)); + multiEntryForm.put("key", "value"); + Assertions.assertEquals("abc=def&123=456&key=value", Utils.formMapToxWwwFormUrlencoded(multiEntryForm)); + } + + @Test + void givenUriAndForm_whenConvertingToHttpRequest_thenCorrectResultReturned() { + URI uri = URI.create("https://www.example.com"); + Map form = Map.of("abc", "defgh"); + + HttpRequest request = Utils.formToHttpRequest(uri, form); + + Assertions.assertEquals(uri, request.uri()); + Assertions.assertEquals("POST", request.method()); + HttpHeaders headers = request.headers(); + List contentTypeHeader = headers.allValues("Content-Type"); + Assertions.assertEquals(1, contentTypeHeader.size()); + Assertions.assertEquals("application/x-www-form-urlencoded", contentTypeHeader.getFirst()); + int contentLength = Utils.formMapToxWwwFormUrlencoded(form).length(); + Assertions.assertEquals(contentLength, request.bodyPublisher().orElseThrow().contentLength()); + } +} diff --git a/backend-postgrest/Dockerfile b/backend-postgrest/Dockerfile new file mode 100644 index 0000000..227c29e --- /dev/null +++ b/backend-postgrest/Dockerfile @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: 2021 - 2025 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2021 - 2025 Netherlands eScience Center +# SPDX-FileCopyrightText: 2023 Dusan Mijatovic (Netherlands eScience Center) +# +# SPDX-License-Identifier: Apache-2.0 + +FROM postgrest/postgrest:v12.2.8 diff --git a/background-services/Dockerfile b/background-services/Dockerfile new file mode 100644 index 0000000..1a7ef97 --- /dev/null +++ b/background-services/Dockerfile @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2025 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +FROM maven:3.9.8-eclipse-temurin-21 AS builder +WORKDIR /usr/mymaven +RUN mkdir src +COPY pom.xml . +RUN mvn dependency:go-offline +COPY ./src ./src +RUN mvn package --offline + +FROM eclipse-temurin:21-jre-jammy +WORKDIR /usr/myjava +COPY --from=builder /usr/mymaven/target/*-jar-with-dependencies.jar background-services.jar +CMD java -cp background-services.jar nl.esciencecenter.rsd.MainDatabase diff --git a/background-services/README.md b/background-services/README.md new file mode 100644 index 0000000..34d6af3 --- /dev/null +++ b/background-services/README.md @@ -0,0 +1,20 @@ + + +# Background services module + +This module currently auto-updates the materialized view `count_software_mentions_cached` every five minutes. Look in `.env.example` (search for `background-services`) to see what environment variables need to be set. + +## Developing + +To run the program locally, remember to load in the env variables and override the value of `POSTGRES_DB_HOST` to `localhost`. + +Please respect the current code style (e.g. tabs for indentation) and prevent your IDE/editor to make style changes to existing code. The code is currently formatted using IntelliJ IDEA. + +## Future possibilities + +This module could be expanded to allow admins to schedule arbitrary database queries, e.g. to auto-clean dangling images and mentions. diff --git a/background-services/pom.xml b/background-services/pom.xml new file mode 100644 index 0000000..dd1f845 --- /dev/null +++ b/background-services/pom.xml @@ -0,0 +1,83 @@ + + + + + + 4.0.0 + + nl.esciencecenter + background-services + 1.0-SNAPSHOT + + + 21 + 21 + UTF-8 + research-software-directory + https://sonarcloud.io + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.7.1 + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.7.1 + + + + jar-with-dependencies + + + + + + package + + single + + + + + + + + + + + org.postgresql + postgresql + 42.7.5 + + + + + org.slf4j + slf4j-api + 2.0.16 + + + + + ch.qos.logback + logback-classic + 1.5.12 + + + + diff --git a/background-services/src/main/java/nl/esciencecenter/rsd/MainDatabase.java b/background-services/src/main/java/nl/esciencecenter/rsd/MainDatabase.java new file mode 100644 index 0000000..449b000 --- /dev/null +++ b/background-services/src/main/java/nl/esciencecenter/rsd/MainDatabase.java @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +public class MainDatabase { + + private static final String VIEW_NAME = "count_software_mentions_cached"; + + public static void main(String[] args) { + Logger logger = LoggerFactory.getLogger(MainDatabase.class); + + Runnable runnable = () -> { + String dbHost = System.getenv("POSTGRES_DB_HOST"); + String dbPort = System.getenv("POSTGRES_DB_HOST_PORT"); + String dbName = System.getenv("POSTGRES_DB"); + String dbUrl = "jdbc:postgresql://%s:%s/%s".formatted(dbHost, dbPort, dbName); + String dbUsername = System.getenv("POSTGRES_USER"); + String dbPassword = System.getenv("POSTGRES_PASSWORD"); + try ( + Connection connection = DriverManager.getConnection(dbUrl, dbUsername, dbPassword); + Statement statement = connection.createStatement() + ) { + statement.execute("REFRESH MATERIALIZED VIEW CONCURRENTLY %s;".formatted(VIEW_NAME)); + } catch (SQLException e) { + throw new RuntimeException(e); + } + }; + + try (ScheduledExecutorService runner = Executors.newSingleThreadScheduledExecutor()) { + int initialDelay = 30; + int delay = 300; + ScheduledFuture future = runner.scheduleWithFixedDelay(runnable, initialDelay, delay, TimeUnit.SECONDS); + future.get(); + } catch (ExecutionException e) { + logger.error("Something went wrong updating the materialized view \"%s\"".formatted(VIEW_NAME), e); + } catch (InterruptedException e) { + logger.error("Got interrupted when updating the materialized view \"%s\"".formatted(VIEW_NAME), e); + Thread.currentThread().interrupt(); + } + } +} diff --git a/background-services/src/main/resources/logback.xml b/background-services/src/main/resources/logback.xml new file mode 100644 index 0000000..1030ab4 --- /dev/null +++ b/background-services/src/main/resources/logback.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} -%kvp- %msg%n + + + + + + + diff --git a/database/.editorconfig b/database/.editorconfig new file mode 100644 index 0000000..f4b6b5f --- /dev/null +++ b/database/.editorconfig @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2024 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +# EditorConfig is awesome: https://editorconfig.org + +# Unix-style newlines with a newline ending every file and tab indentation +[**] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true +indent_style = tab + +[**.md] +indent_style = unset diff --git a/database/000-fill-authenticator-password.sh b/database/000-fill-authenticator-password.sh new file mode 100644 index 0000000..07d9894 --- /dev/null +++ b/database/000-fill-authenticator-password.sh @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: 2022 - 2025 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2022 - 2025 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +sed --in-place "s/POSTGRES_AUTHENTICATOR_PASSWORD/$(echo $POSTGRES_AUTHENTICATOR_PASSWORD)/1" /docker-entrypoint-initdb.d/001-setup-basic-roles.sql +sed --in-place "s/_database_name_/$(echo $POSTGRES_DB)/1" /docker-entrypoint-initdb.d/001-setup-basic-roles.sql diff --git a/database/001-setup-basic-roles.sql b/database/001-setup-basic-roles.sql new file mode 100644 index 0000000..b51f94c --- /dev/null +++ b/database/001-setup-basic-roles.sql @@ -0,0 +1,28 @@ +-- SPDX-FileCopyrightText: 2021 - 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2021 - 2025 Netherlands eScience Center +-- +-- SPDX-License-Identifier: Apache-2.0 + +ALTER DATABASE "_database_name_" SET JIT = OFF; + +CREATE ROLE rsd_authenticator NOINHERIT LOGIN PASSWORD 'POSTGRES_AUTHENTICATOR_PASSWORD'; + +CREATE ROLE rsd_web_anon NOLOGIN; + +GRANT USAGE ON SCHEMA public TO rsd_web_anon; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO rsd_web_anon; + +GRANT rsd_web_anon TO rsd_authenticator; + +CREATE ROLE rsd_admin NOLOGIN; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO rsd_admin; + +GRANT rsd_admin TO rsd_authenticator; + +CREATE ROLE rsd_user NOLOGIN; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO rsd_user; + +GRANT rsd_user TO rsd_authenticator; diff --git a/database/002-create-account-table.sql b/database/002-create-account-table.sql new file mode 100644 index 0000000..f820cbf --- /dev/null +++ b/database/002-create-account-table.sql @@ -0,0 +1,143 @@ +-- SPDX-FileCopyrightText: 2021 - 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2021 - 2025 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2022 - 2023 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2022 - 2023 dv4all +-- SPDX-FileCopyrightText: 2023 - 2025 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2023 Christian Meeßen (GFZ) +-- SPDX-FileCopyrightText: 2023 Dusan Mijatovic (dv4all) (dv4all) +-- SPDX-FileCopyrightText: 2023 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE TABLE account ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + agree_terms BOOLEAN DEFAULT FALSE NOT NULL, + agree_terms_updated_at TIMESTAMPTZ, + notice_privacy_statement BOOLEAN DEFAULT FALSE NOT NULL, + notice_privacy_statement_updated_at TIMESTAMPTZ +); + +CREATE FUNCTION sanitise_insert_account() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + NEW.agree_terms_updated_at = NEW.created_at; + NEW.notice_privacy_statement_updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_account BEFORE INSERT ON account FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_account(); + + +CREATE FUNCTION sanitise_update_account() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + IF NEW.agree_terms != OLD.agree_terms THEN + NEW.agree_terms_updated_at = NEW.updated_at; + ELSE + NEW.agree_terms_updated_at = OLD.agree_terms_updated_at; + END IF; + IF NEW.notice_privacy_statement != OLD.notice_privacy_statement THEN + NEW.notice_privacy_statement_updated_at = NEW.updated_at; + ELSE + NEW.notice_privacy_statement_updated_at = OLD.notice_privacy_statement_updated_at; + END IF; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_account BEFORE UPDATE ON account FOR EACH ROW EXECUTE PROCEDURE sanitise_update_account(); + + +CREATE TABLE login_for_account ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + account UUID REFERENCES account (id) NOT NULL, + provider VARCHAR(100) NOT NULL, + sub VARCHAR(50) NOT NULL, + name VARCHAR(200), + email VARCHAR(200), + home_organisation VARCHAR(200), + last_login_date TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + UNIQUE(provider, sub) +); + +CREATE INDEX login_for_account_account_idx ON login_for_account(account); + +CREATE FUNCTION sanitise_insert_login_for_account() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_login_for_account BEFORE INSERT ON login_for_account FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_login_for_account(); + + +CREATE FUNCTION sanitise_update_login_for_account() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_login_for_account BEFORE UPDATE ON login_for_account FOR EACH ROW EXECUTE PROCEDURE sanitise_update_login_for_account(); + + +CREATE TABLE admin_account ( + account_id UUID REFERENCES account (id) PRIMARY KEY +); + + +CREATE TABLE account_invite ( + id UUID PRIMARY KEY, + uses_left INTEGER, --NULL means bulk (i.e. infinite use) invite + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE FUNCTION sanitise_insert_account_invite() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_account_invite BEFORE INSERT ON account_invite FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_account_invite(); + +CREATE FUNCTION sanitise_update_account_invite() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_account_invite BEFORE UPDATE ON account_invite FOR EACH ROW EXECUTE PROCEDURE sanitise_update_login_for_account(); + +ALTER TABLE account_invite ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_all_rights ON account_invite TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); diff --git a/database/003-create-image-table.sql b/database/003-create-image-table.sql new file mode 100644 index 0000000..001b4cb --- /dev/null +++ b/database/003-create-image-table.sql @@ -0,0 +1,88 @@ +-- SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2022 dv4all +-- SPDX-FileCopyrightText: 2023 Dusan Mijatovic (Netherlands eScience Center) +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- create extension pgcrypto to encrypt base64 content +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +CREATE TABLE image ( + id VARCHAR(40) PRIMARY KEY, + data VARCHAR(2750000) NOT NULL, + mime_type VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + + CONSTRAINT image_valid_mime_type CHECK (mime_type IN ( + 'image/avif', + 'image/gif', + 'image/jpeg', + 'image/png', + 'image/svg+xml', + 'image/webp', + 'image/x-icon' + )) +); + +CREATE FUNCTION sanitise_insert_image() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN +-- create SHA-1 id based on provided data content + NEW.id = ENCODE(DIGEST(NEW.data, 'sha1'), 'hex'); + NEW.created_at = LOCALTIMESTAMP; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_image BEFORE INSERT ON image FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_image(); + +CREATE FUNCTION sanitise_update_image() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN +-- create SHA-1 id based on provided data content + NEW.id = ENCODE(DIGEST(NEW.data, 'sha1'), 'hex'); + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_image BEFORE UPDATE ON image FOR EACH ROW EXECUTE PROCEDURE sanitise_update_image(); + + +-- ---------------------------------------- +-- RPC to get image by id => sha-1 of data +-- cache incrased to 1 year based on lighthouse audit +-- ---------------------------------------- + +CREATE DOMAIN "application/octet-stream" AS BYTEA; + +CREATE FUNCTION get_image(uid VARCHAR(40)) RETURNS "application/octet-stream" STABLE LANGUAGE plpgsql AS +$$ +DECLARE headers TEXT; +DECLARE blob BYTEA; + +BEGIN + SELECT format( + '[{"Content-Type": "%s"},' + '{"Content-Disposition": "inline; filename=\"%s\""},' + '{"Cache-Control": "max-age=31536001"}]', + mime_type, + uid) + FROM image WHERE id = uid INTO headers; + + PERFORM set_config('response.headers', headers, TRUE); + + SELECT decode(image.data, 'base64') FROM image WHERE id = uid INTO blob; + + IF FOUND + THEN RETURN(blob); + ELSE RAISE SQLSTATE 'PT404' + USING + message = 'NOT FOUND', + detail = 'File not found', + hint = format('%s seems to be an invalid file id', image_id); + END IF; +END +$$; + diff --git a/database/004-create-software-table.sql b/database/004-create-software-table.sql new file mode 100644 index 0000000..7146f54 --- /dev/null +++ b/database/004-create-software-table.sql @@ -0,0 +1,68 @@ +-- SPDX-FileCopyrightText: 2021 - 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2021 - 2025 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2022 - 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +-- SPDX-FileCopyrightText: 2022 Christian Meeßen (GFZ) +-- SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2022 dv4all +-- SPDX-FileCopyrightText: 2023 Diego Alonso Alvarez (ICL) +-- SPDX-FileCopyrightText: 2023 Imperial College London +-- SPDX-FileCopyrightText: 2025 Paula Stock (GFZ) +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- install citext extension for +-- case insensitive indexing +-- https://www.postgresql.org/docs/current/citext.html +CREATE EXTENSION IF NOT EXISTS citext; + +CREATE TYPE description_type AS ENUM ( + 'link', + 'markdown' +); + +CREATE TABLE software ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + slug VARCHAR(250) UNIQUE NOT NULL CHECK (slug ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + brand_name VARCHAR(250) NOT NULL, + closed_source BOOLEAN DEFAULT FALSE NOT NULL, + concept_doi CITEXT CHECK (concept_doi ~ '^10(\.\w+)+/\S+$' AND LENGTH(concept_doi) <= 255), + description VARCHAR(10000), + description_url VARCHAR(200) CHECK (description_url ~ '^https?://\S+$'), + description_type description_type DEFAULT 'markdown' NOT NULL, + get_started_url VARCHAR(200) CHECK (get_started_url ~ '^https?://\S+$'), + image_id VARCHAR(40) REFERENCES image(id), + is_published BOOLEAN DEFAULT FALSE NOT NULL, + short_statement VARCHAR(300), + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE FUNCTION sanitise_insert_software() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_software BEFORE INSERT ON software FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_software(); + + +CREATE FUNCTION sanitise_update_software() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + + IF NEW.slug IS DISTINCT FROM OLD.slug AND CURRENT_USER IS DISTINCT FROM 'rsd_admin' AND (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) IS DISTINCT FROM TRUE THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to change the slug'; + END IF; + + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_software BEFORE UPDATE ON software FOR EACH ROW EXECUTE PROCEDURE sanitise_update_software(); diff --git a/database/005-create-relations-for-software.sql b/database/005-create-relations-for-software.sql new file mode 100644 index 0000000..438805d --- /dev/null +++ b/database/005-create-relations-for-software.sql @@ -0,0 +1,270 @@ +-- SPDX-FileCopyrightText: 2021 - 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2021 - 2025 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2022 - 2023 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2022 - 2024 dv4all +-- SPDX-FileCopyrightText: 2022 - 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +-- SPDX-FileCopyrightText: 2022 Christian Meeßen (GFZ) +-- SPDX-FileCopyrightText: 2023 - 2025 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2024 Dusan Mijatovic (dv4all) (dv4all) +-- SPDX-FileCopyrightText: 2025 Paula Stock (GFZ) +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE TYPE platform_type AS ENUM ( + 'github', + 'gitlab', + 'bitbucket', + '4tu', + 'other' +); + +CREATE TABLE repository_url ( + software UUID REFERENCES software (id) PRIMARY KEY, + url VARCHAR(200) NOT NULL CHECK (url ~ '^https?://\S+$'), + code_platform platform_type NOT NULL DEFAULT 'other', + archived BOOLEAN, + license VARCHAR(200), + star_count BIGINT, + fork_count INTEGER, + open_issue_count INTEGER, + basic_data_last_error VARCHAR(500), + basic_data_scraped_at TIMESTAMPTZ, + languages JSONB, + languages_last_error VARCHAR(500), + languages_scraped_at TIMESTAMPTZ, + commit_history JSONB, + commit_history_last_error VARCHAR(500), + commit_history_scraped_at TIMESTAMPTZ, + contributor_count INTEGER, + contributor_count_last_error VARCHAR(500), + contributor_count_scraped_at TIMESTAMPTZ, + scraping_disabled_reason VARCHAR(200) +); + +-- NOTE! When changing package_manager_type +-- please update packageManagerSettings const +-- in the frontend file apiPackageManager.ts +CREATE TYPE package_manager_type AS ENUM ( + 'anaconda', + 'chocolatey', + 'cran', + 'crates', + 'debian', + 'dockerhub', + 'fourtu', + 'ghcr', + 'github', + 'gitlab', + 'golang', + 'maven', + 'npm', + 'pixi', + 'pypi', + 'snapcraft', + 'sonatype', + 'other' +); + +CREATE TABLE package_manager ( + id UUID PRIMARY KEY, + software UUID REFERENCES software (id) NOT NULL, + url VARCHAR(200) NOT NULL CHECK (url ~ '^https?://\S+$'), + package_manager package_manager_type NOT NULL DEFAULT 'other', + UNIQUE(software, url, package_manager), + download_count BIGINT, + download_count_last_error VARCHAR(500), + download_count_scraped_at TIMESTAMPTZ, + download_count_scraping_disabled_reason VARCHAR(200), + reverse_dependency_count INTEGER, + reverse_dependency_count_last_error VARCHAR(500), + reverse_dependency_count_scraped_at TIMESTAMPTZ, + reverse_dependency_count_scraping_disabled_reason VARCHAR(200), + position INTEGER, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE FUNCTION sanitise_insert_package_manager() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_package_manager BEFORE INSERT ON package_manager FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_package_manager(); + +-- We do not allow update of url +CREATE FUNCTION sanitise_update_package_manager() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.software = old.software; + -- NEW.url = old.url; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_package_manager BEFORE UPDATE ON package_manager FOR EACH ROW EXECUTE PROCEDURE sanitise_update_package_manager(); + + + +CREATE TABLE license_for_software ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + software UUID REFERENCES software (id) NOT NULL, + license VARCHAR(100) NOT NULL, + name VARCHAR(200) NULL, + reference VARCHAR(200) NULL CHECK (reference ~ '^https?://\S+$'), + open_source BOOLEAN NOT NULL DEFAULT TRUE, + UNIQUE(software, license), + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE FUNCTION sanitise_insert_license_for_software() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_license_for_software BEFORE INSERT ON license_for_software FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_license_for_software(); + + +CREATE FUNCTION sanitise_update_license_for_software() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_license_for_software BEFORE UPDATE ON license_for_software FOR EACH ROW EXECUTE PROCEDURE sanitise_update_license_for_software(); + + + +CREATE TABLE contributor ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + software UUID REFERENCES software (id) NOT NULL, + is_contact_person BOOLEAN NOT NULL DEFAULT FALSE, + email_address VARCHAR(200), + family_names VARCHAR(200) NOT NULL, + given_names VARCHAR(200) NOT NULL, + affiliation VARCHAR(200) CHECK (affiliation ~ '^\S+( \S+)*$'), + role VARCHAR(200) CHECK (role ~ '^\S+( \S+)*$'), + orcid VARCHAR(19) CHECK (orcid ~ '^\d{4}-\d{4}-\d{4}-\d{3}[0-9X]$'), + position INTEGER, + avatar_id VARCHAR(40) REFERENCES image(id), + -- support for (loosely) linking of user_profile entry without ORCID + account UUID REFERENCES account (id), + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX contributor_software_idx ON contributor(software); +CREATE INDEX contributor_orcid_idx ON contributor(orcid); +CREATE INDEX contributor_account_idx ON contributor(account); + +CREATE FUNCTION sanitise_insert_contributor() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_contributor BEFORE INSERT ON contributor FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_contributor(); + + +CREATE FUNCTION sanitise_update_contributor() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + IF OLD.account IS NOT NULL AND (CURRENT_USER = 'rsd_admin' OR (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER)) IS DISTINCT FROM TRUE THEN + NEW.family_names = OLD.family_names; + NEW.given_names = OLD.given_names; + NEW.orcid = OLD.orcid; + END IF; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_contributor BEFORE UPDATE ON contributor FOR EACH ROW EXECUTE PROCEDURE sanitise_update_contributor(); + +CREATE TABLE testimonial ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + software UUID REFERENCES software (id) NOT NULL, + message VARCHAR(500) NOT NULL, + source VARCHAR(200) NOT NULL, + position INTEGER +); + +CREATE INDEX testimonial_software_idx ON testimonial(software); + +CREATE FUNCTION sanitise_insert_testimonial() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_testimonial BEFORE INSERT ON testimonial FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_testimonial(); + + +CREATE FUNCTION sanitise_update_testimonial() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_testimonial BEFORE UPDATE ON testimonial FOR EACH ROW EXECUTE PROCEDURE sanitise_update_testimonial(); + + + +CREATE TABLE software_highlight ( + software UUID REFERENCES software (id) PRIMARY KEY, + date_start DATE, + date_end DATE, + position INTEGER, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE FUNCTION sanitise_insert_software_highlight() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_software_highlight BEFORE INSERT ON software_highlight FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_software_highlight(); + +CREATE FUNCTION sanitise_update_software_highlight() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.software = OLD.software; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_software_highlight BEFORE UPDATE ON software_highlight FOR EACH ROW EXECUTE PROCEDURE sanitise_update_software_highlight(); diff --git a/database/006-create-project-table.sql b/database/006-create-project-table.sql new file mode 100644 index 0000000..1fb430c --- /dev/null +++ b/database/006-create-project-table.sql @@ -0,0 +1,85 @@ +-- SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2022 dv4all +-- SPDX-FileCopyrightText: 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +-- SPDX-FileCopyrightText: 2025 Paula Stock (GFZ) +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE TABLE project ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + slug VARCHAR(200) UNIQUE NOT NULL CHECK (slug ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + title VARCHAR(200) NOT NULL, + subtitle VARCHAR(300), + date_end DATE, + date_start DATE, + description VARCHAR(10000), + grant_id VARCHAR(50), + image_caption VARCHAR(500), + image_contain BOOLEAN DEFAULT FALSE NOT NULL, + image_id VARCHAR(40) REFERENCES image(id), + is_published BOOLEAN DEFAULT FALSE NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE FUNCTION sanitise_insert_project() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_project BEFORE INSERT ON project FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_project(); + + +CREATE FUNCTION sanitise_update_project() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + + IF NEW.slug IS DISTINCT FROM OLD.slug AND CURRENT_USER IS DISTINCT FROM 'rsd_admin' AND (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) IS DISTINCT FROM TRUE THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to change the slug'; + END IF; + + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_project BEFORE UPDATE ON project FOR EACH ROW EXECUTE PROCEDURE sanitise_update_project(); + + +CREATE TABLE url_for_project ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + project UUID REFERENCES project (id), + title VARCHAR(100) NOT NULL, + url VARCHAR(200) NOT NULL CHECK (url ~ '^https?://\S+$'), + position INTEGER +); + +CREATE FUNCTION sanitise_insert_url_for_project() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_url_for_project BEFORE INSERT ON url_for_project FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_url_for_project(); + + +CREATE FUNCTION sanitise_update_url_for_project() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_url_for_project BEFORE UPDATE ON url_for_project FOR EACH ROW EXECUTE PROCEDURE sanitise_update_url_for_project(); diff --git a/database/007-create-relations-for-projects.sql b/database/007-create-relations-for-projects.sql new file mode 100644 index 0000000..8c37466 --- /dev/null +++ b/database/007-create-relations-for-projects.sql @@ -0,0 +1,91 @@ +-- SPDX-FileCopyrightText: 2022 - 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2022 - 2025 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2022 dv4all +-- SPDX-FileCopyrightText: 2024 - 2025 Dusan Mijatovic (Netherlands eScience Center) +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE TABLE team_member ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + project UUID REFERENCES project (id) NOT NULL, + is_contact_person BOOLEAN NOT NULL DEFAULT FALSE, + email_address VARCHAR(200), + family_names VARCHAR(200) NOT NULL, + given_names VARCHAR(200) NOT NULL, + affiliation VARCHAR(200) CHECK (affiliation ~ '^\S+( \S+)*$'), + role VARCHAR(200) CHECK (role ~ '^\S+( \S+)*$'), + orcid VARCHAR(19) CHECK (orcid ~ '^\d{4}-\d{4}-\d{4}-\d{3}[0-9X]$'), + position INTEGER, + avatar_id VARCHAR(40) REFERENCES image(id), + -- support for (loosely) linking of user_profile entry without ORCID + account UUID REFERENCES account (id), + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX team_member_project_idx ON team_member(project); +CREATE INDEX team_member_orcid_idx ON team_member(orcid); +CREATE INDEX team_member_account_idx ON team_member(account); + +CREATE FUNCTION sanitise_insert_team_member() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_team_member BEFORE INSERT ON team_member FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_team_member(); + + +CREATE FUNCTION sanitise_update_team_member() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + IF OLD.account IS NOT NULL AND (CURRENT_USER = 'rsd_admin' OR (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER)) IS DISTINCT FROM TRUE THEN + NEW.family_names = OLD.family_names; + NEW.given_names = OLD.given_names; + NEW.orcid = OLD.orcid; + END IF; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_team_member BEFORE UPDATE ON team_member FOR EACH ROW EXECUTE PROCEDURE sanitise_update_team_member(); + + +CREATE TABLE testimonial_for_project ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + project UUID REFERENCES project (id) NOT NULL, + message VARCHAR(500) NOT NULL, + source VARCHAR(200) NOT NULL, + position INTEGER +); + +CREATE INDEX testimonial_for_project_project_idx ON testimonial_for_project(project); + +CREATE FUNCTION sanitise_insert_testimonial_for_project() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_testimonial_for_project BEFORE INSERT ON testimonial_for_project FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_testimonial_for_project(); + + +CREATE FUNCTION sanitise_update_testimonial_for_project() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_testimonial_for_project BEFORE UPDATE ON testimonial_for_project FOR EACH ROW EXECUTE PROCEDURE sanitise_update_testimonial_for_project(); diff --git a/database/008-create-community-table.sql b/database/008-create-community-table.sql new file mode 100644 index 0000000..e46676c --- /dev/null +++ b/database/008-create-community-table.sql @@ -0,0 +1,351 @@ +-- SPDX-FileCopyrightText: 2024 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2024 Netherlands eScience Center +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE TABLE community ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + slug VARCHAR(200) UNIQUE NOT NULL CHECK (slug ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + name VARCHAR(200) NOT NULL, + short_description VARCHAR(300), + description VARCHAR(10000), + primary_maintainer UUID REFERENCES account (id), + logo_id VARCHAR(40) REFERENCES image(id), + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +-- SANITISE insert and update +-- ONLY rsd_admin and primary_maintainer can change community table +-- ONLY rsd_admin can change primary_maintainer value +CREATE FUNCTION sanitise_insert_community() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + + IF CURRENT_USER = 'rsd_admin' OR (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) THEN + RETURN NEW; + END IF; + + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to add this community'; +END +$$; + +CREATE TRIGGER sanitise_insert_community BEFORE INSERT ON community FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_community(); + + +CREATE FUNCTION sanitise_update_community() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + + IF NEW.slug IS DISTINCT FROM OLD.slug AND CURRENT_USER IS DISTINCT FROM 'rsd_admin' AND (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) IS DISTINCT FROM TRUE THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to change the slug'; + END IF; + + IF NEW.primary_maintainer IS DISTINCT FROM OLD.primary_maintainer AND CURRENT_USER IS DISTINCT FROM 'rsd_admin' AND (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) IS DISTINCT FROM TRUE THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to change the primary maintainer for community ' || OLD.name; + END IF; + + RETURN NEW; +END +$$; + +CREATE TRIGGER sanitise_update_community BEFORE UPDATE ON community FOR EACH ROW EXECUTE PROCEDURE sanitise_update_community(); + + +-- MAINTAINER OF COMMUNITY +CREATE TABLE maintainer_for_community ( + maintainer UUID REFERENCES account (id), + community UUID REFERENCES community (id), + PRIMARY KEY (maintainer, community) +); + + +-- Needed for RLS on various tables +CREATE FUNCTION communities_of_current_maintainer() RETURNS SETOF UUID STABLE +LANGUAGE sql SECURITY DEFINER AS +$$ + SELECT + id + FROM + community + WHERE + primary_maintainer = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account') + UNION + SELECT + community + FROM + maintainer_for_community + WHERE + maintainer = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'); +$$; + + +-- RLS community table +ALTER TABLE community ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON community FOR SELECT TO rsd_web_anon, rsd_user + USING (TRUE); + +CREATE POLICY maintainer_all_rights ON community TO rsd_user + USING (id IN (SELECT * FROM communities_of_current_maintainer())) + WITH CHECK (TRUE); + +CREATE POLICY admin_all_rights ON community TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- RLS maintainer_for_community table +ALTER TABLE maintainer_for_community ENABLE ROW LEVEL SECURITY; + +CREATE POLICY maintainer_select ON maintainer_for_community FOR SELECT TO rsd_user + USING (community IN (SELECT * FROM communities_of_current_maintainer())); + +CREATE POLICY maintainer_delete ON maintainer_for_community FOR DELETE TO rsd_user + USING (community IN (SELECT * FROM communities_of_current_maintainer())); + +CREATE POLICY maintainer_insert ON maintainer_for_community FOR INSERT TO rsd_user + WITH CHECK (community IN (SELECT * FROM communities_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON maintainer_for_community TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- INVITES FOR COMMUNITY MAINTAINER (magic link) +CREATE TABLE invite_maintainer_for_community ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + community UUID REFERENCES community (id) NOT NULL, + created_by UUID REFERENCES account (id), + claimed_by UUID REFERENCES account (id), + claimed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT LOCALTIMESTAMP, + expires_at TIMESTAMPTZ NOT NULL GENERATED ALWAYS AS (created_at AT TIME ZONE 'UTC' + INTERVAL '31 days') STORED +); + +CREATE FUNCTION sanitise_insert_invite_maintainer_for_community() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.claimed_by = NULL; + NEW.claimed_at = NULL; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_invite_maintainer_for_community BEFORE INSERT ON invite_maintainer_for_community FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_invite_maintainer_for_community(); + +CREATE FUNCTION sanitise_update_invite_maintainer_for_community() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.community = OLD.community; + NEW.created_by = OLD.created_by; + NEW.created_at = OLD.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_invite_maintainer_for_community BEFORE UPDATE ON invite_maintainer_for_community FOR EACH ROW EXECUTE PROCEDURE sanitise_update_invite_maintainer_for_community(); + +-- RLS invite_maintainer_for_community table +ALTER TABLE invite_maintainer_for_community ENABLE ROW LEVEL SECURITY; + +CREATE POLICY maintainer_select ON invite_maintainer_for_community FOR SELECT TO rsd_user + USING (community IN (SELECT * FROM communities_of_current_maintainer()) + OR created_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account') + OR claimed_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +CREATE POLICY maintainer_delete ON invite_maintainer_for_community FOR DELETE TO rsd_user + USING (community IN (SELECT * FROM communities_of_current_maintainer()) + OR created_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account') + OR claimed_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +CREATE POLICY maintainer_insert ON invite_maintainer_for_community FOR INSERT TO rsd_user + WITH CHECK (community IN (SELECT * FROM communities_of_current_maintainer()) AND created_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +CREATE POLICY admin_all_rights ON invite_maintainer_for_community TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- COMMUNITY maintainers list with basic personal info +-- used in the community maintainers page +CREATE FUNCTION maintainers_of_community(community_id UUID) RETURNS TABLE ( + maintainer UUID, + name VARCHAR[], + email VARCHAR[], + affiliation VARCHAR[], + is_primary BOOLEAN +) LANGUAGE plpgsql STABLE SECURITY DEFINER AS +$$ +DECLARE account_authenticated UUID; +BEGIN + account_authenticated = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'); + IF account_authenticated IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please login first'; + END IF; + + IF community_id IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please provide a community id'; + END IF; + + IF NOT community_id IN (SELECT * FROM communities_of_current_maintainer()) AND + CURRENT_USER IS DISTINCT FROM 'rsd_admin' AND ( + SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER + ) IS DISTINCT FROM TRUE THEN + RAISE EXCEPTION USING MESSAGE = 'You are not a maintainer of this community'; + END IF; + + RETURN QUERY + WITH maintainer_ids AS ( + -- primary maintainer of community + SELECT + community.primary_maintainer AS maintainer, + TRUE AS is_primary + FROM + community + WHERE + community.id = community_id + -- append second selection + UNION ALL + -- other maintainers of community + SELECT + maintainer_for_community.maintainer, + FALSE AS is_primary + FROM + maintainer_for_community + WHERE + maintainer_for_community.community = community_id + -- primary as first record + ORDER BY is_primary DESC + ) + SELECT + maintainer_ids.maintainer AS maintainer, + ARRAY_AGG(login_for_account."name") AS name, + ARRAY_AGG(login_for_account.email) AS email, + ARRAY_AGG(login_for_account.home_organisation) AS affiliation, + BOOL_OR(maintainer_ids.is_primary) AS is_primary + FROM + maintainer_ids + INNER JOIN + login_for_account ON login_for_account.account = maintainer_ids.maintainer + GROUP BY + maintainer_ids.maintainer + -- primary as first record + ORDER BY + is_primary DESC; + RETURN; +END +$$; + +-- ACCEPT MAGIC LINK INVITATION +-- REGISTER user with this link as maintainer of community +-- RETURN basic info about community on SUCCESS +CREATE FUNCTION accept_invitation_community(invitation UUID) RETURNS TABLE( + id UUID, + name VARCHAR, + slug VARCHAR +) LANGUAGE plpgsql VOLATILE SECURITY DEFINER AS +$$ +DECLARE invitation_row invite_maintainer_for_community%ROWTYPE; +DECLARE account UUID; +BEGIN + account = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'); + IF account IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please login first'; + END IF; + + IF invitation IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please provide an invitation id'; + END IF; + + SELECT * FROM + invite_maintainer_for_community + WHERE + invite_maintainer_for_community.id = invitation INTO invitation_row; + + IF invitation_row.id IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Invitation with id ' || invitation || ' does not exist'; + END IF; + + IF invitation_row.claimed_by IS NOT NULL OR invitation_row.claimed_at IS NOT NULL OR + invitation_row.expires_at < CURRENT_TIMESTAMP THEN + RAISE EXCEPTION USING MESSAGE = 'Invitation with id ' || invitation || ' is expired'; + END IF; + +-- Only use the invitation if not already a maintainer + IF NOT EXISTS( + SELECT + maintainer_for_community.maintainer + FROM + maintainer_for_community + WHERE + maintainer_for_community.maintainer=account AND maintainer_for_community.community=invitation_row.community + UNION + SELECT + community.primary_maintainer AS maintainer + FROM + community + WHERE + community.primary_maintainer=account AND community.id=invitation_row.community + LIMIT 1 + ) THEN + + UPDATE invite_maintainer_for_community + SET claimed_by = account, claimed_at = LOCALTIMESTAMP + WHERE invite_maintainer_for_community.id = invitation; + + INSERT INTO maintainer_for_community + VALUES (account, invitation_row.community); + + END IF; + + RETURN QUERY + SELECT + community.id, + community.name, + community.slug + FROM + community + WHERE + community.id = invitation_row.community; + RETURN; +END +$$; + + +-- SOFTWARE FOR COMMUNITY +-- request status of software to be added to community +-- default value is pending +CREATE TYPE request_status AS ENUM ( + 'pending', + 'approved', + 'rejected' +); + +CREATE TABLE software_for_community ( + software UUID REFERENCES software (id), + community UUID REFERENCES community (id), + status request_status NOT NULL DEFAULT 'pending', + PRIMARY KEY (software, community) +); + +CREATE FUNCTION sanitise_update_software_for_community() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.software = OLD.software; + NEW.community = OLD.community; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_software_for_community BEFORE UPDATE ON software_for_community FOR EACH ROW EXECUTE PROCEDURE sanitise_update_software_for_community(); diff --git a/database/010-create-research-domain.sql b/database/010-create-research-domain.sql new file mode 100644 index 0000000..c1d3422 --- /dev/null +++ b/database/010-create-research-domain.sql @@ -0,0 +1,515 @@ +-- SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2022 dv4all +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- RESEARCH DOMAINS + +-- create table for RSE research domains +CREATE TABLE research_domain ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + key VARCHAR(50) UNIQUE NOT NULL, + name VARCHAR(200), + description VARCHAR(500), + parent UUID REFERENCES research_domain (id) +); + +CREATE FUNCTION check_cycle_research_domain() RETURNS TRIGGER STABLE LANGUAGE plpgsql SECURITY DEFINER AS +$$ +DECLARE initial_rd UUID = NEW.id; +DECLARE current_rd UUID = NEW.parent; +BEGIN + WHILE current_rd IS NOT NULL LOOP + IF current_rd = initial_rd THEN + RAISE EXCEPTION USING MESSAGE = 'Cycle detected for research domain with id ' || NEW.id; + END IF; + SELECT parent FROM research_domain WHERE id = current_rd INTO current_rd; + END LOOP; + RETURN NEW; +END +$$; + +-- z_ prefix so that if is executed after the sanitise_update_organisation trigger +CREATE TRIGGER z_check_cycle_research_domain BEFORE UPDATE OF parent ON research_domain FOR EACH ROW EXECUTE PROCEDURE check_cycle_research_domain(); + + +CREATE FUNCTION sanitise_insert_research_domain() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_research_domain BEFORE INSERT ON research_domain FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_research_domain(); + + +CREATE FUNCTION sanitise_update_research_domain() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_research_domain BEFORE UPDATE ON research_domain FOR EACH ROW EXECUTE PROCEDURE sanitise_update_research_domain(); + + +-- create reference table for projects +CREATE TABLE research_domain_for_project ( + project UUID REFERENCES project (id), + research_domain UUID REFERENCES research_domain (id), + PRIMARY KEY (project, research_domain) +); + +CREATE INDEX research_domain_for_project_research_domain_idx ON research_domain_for_project(research_domain); + + +-- insert RSE list of research domains + +-- level 1 +INSERT INTO research_domain (key, name, description, parent) VALUES + ('PE', 'Physical Sciences and Engineering', 'Physical Sciences and Engineering (PE)', null), + ('LS', 'Life Sciences', 'Life Sciences (LS)', null), + ('SH', 'Social Sciences and Humanities', 'Social Sciences and Humanities (SH)', null) +; + +-- level 2 +INSERT INTO research_domain (key,name,description,parent) +VALUES + ('PE1', 'Mathematics', 'All areas of mathematics, pure and applied, plus mathematical foundations of computer science,mathematical physics and statistics (PE1)', (SELECT id FROM research_domain WHERE key='PE')), + ('PE2', 'Fundamental Constituents of Matter', 'Particle, nuclear, plasma, atomic, molecular, gas, and optical physics (PE2)', (SELECT id FROM research_domain WHERE key='PE')), + ('PE3', 'Condensed Matter Physics', 'Structure, electronic properties, fluids, nanosciences, biological physics (PE3)', (SELECT id FROM research_domain WHERE key='PE')), + ('PE4', 'Physical and Analytical Chemical Sciences', 'Analytical chemistry, chemical theory, physical chemistry/chemical physics (PE4)', (SELECT id FROM research_domain WHERE key='PE')), + ('PE5', 'Synthetic Chemistry and Materials', 'New materials and new synthetic approaches, structure-properties relations, solid state chemistry, molecular architecture, organic chemistry (PE5)', (SELECT id FROM research_domain WHERE key='PE')), + ('PE6', 'Computer Science and Informatics', 'Informatics and information systems, computer science, scientific computing, intelligent systems (PE6)', (SELECT id FROM research_domain WHERE key='PE')), + ('PE7', 'Systems and Communication Engineering', 'Electrical, electronic, communication, optical and systems engineering (PE7)', (SELECT id FROM research_domain WHERE key='PE')), + ('PE8', 'Products and Processes Engineering', 'Product and process design, chemical, civil, environmental, mechanical, vehicle engineering, energy processes and relevant computational methods (PE8)', (SELECT id FROM research_domain WHERE key='PE')), + ('PE9', 'Universe Sciences', 'Astro-physics/-chemistry/-biology; solar system; planetary systems; stellar, galactic and extragalactic astronomy; cosmology; space sciences; astronomical instrumentation and data (PE9)', (SELECT id FROM research_domain WHERE key='PE')), + ('PE10', 'Earth System Science', 'Physical geography, geology, geophysics, atmospheric sciences, oceanography, climatology, cryology, ecology, global environmental change, biogeochemical cycles, natural resources management (PE10)', (SELECT id FROM research_domain WHERE key='PE')), + ('PE11', 'Materials Engineering', 'Advanced materials development: performance enhancement, modelling, large-scale preparation,modification, tailoring, optimisation, novel and combined use of materials, etc. (PE11)', (SELECT id FROM research_domain WHERE key='PE')), + ('LS1', 'Molecules of Life: Biological Mechanisms, Structures and Functions', 'For all organisms: Molecular biology, biochemistry, structural biology, molecular biophysics, synthetic and chemical biology, drug design, innovative methods and modelling (LS1)', (SELECT id FROM research_domain WHERE key='LS')), + ('LS2', 'Integrative Biology: from Genes and Genomes to Systems', 'For all organisms: Genetics, epigenetics, genomics and other ‘omics studies, bioinformatics, systems biology, genetic diseases, gene editing, innovative methods and modelling, ‘omics for personalised medicine (LS2)', (SELECT id FROM research_domain WHERE key='LS')), + ('LS3', 'Cellular, Developmental and Regenerative Biology', 'For all organisms: Structure and function of the cell, cell-cell communication, embryogenesis, tissue differentiation, organogenesis, growth, development, evolution of development, organoids, stem cells, regeneration, therapeutic approaches (LS3)', (SELECT id FROM research_domain WHERE key='LS')), + ('LS4', 'Physiology in Health, Disease and Ageing', 'Organ and tissue physiology, comparative physiology, physiology of ageing, pathophysiology, inter-organ and tissue communication, endocrinology, nutrition, metabolism, interaction with the microbiome, non-communicable diseases including cancer (and except disorders of the nervous system and immunity-related diseases) (LS4)', (SELECT id FROM research_domain WHERE key='LS')), + ('LS5', 'Neuroscience and Disorders of the Nervous System', 'Nervous system development, homeostasis and ageing, nervous system function and dysfunction, systems neuroscience and modelling, biological basis of cognitive processes and of behaviour, neurological and mental disorders (LS5)', (SELECT id FROM research_domain WHERE key='LS')), + ('LS6', 'Immunity, Infection and Immunotherapy', 'The immune system, related disorders and their mechanisms, biology of infectious agents and infection, biological basis of prevention and treatment of infectious diseases, innovative immunological tools and approaches, including therapies (LS6)', (SELECT id FROM research_domain WHERE key='LS')), + ('LS7', 'Prevention, Diagnosis and Treatment of Human Diseases', 'Medical technologies and tools for prevention, diagnosis and treatment of human diseases, therapeutic approaches and interventions, pharmacology, preventative medicine, epidemiology and public health, digital medicine (LS7)', (SELECT id FROM research_domain WHERE key='LS')), + ('LS8', 'Environmental Biology, Ecology and Evolution', 'For all organisms: Ecology, biodiversity, environmental change, evolutionary biology, behavioural ecology, microbial ecology, marine biology, ecophysiology, theoretical developments and modelling (LS8)', (SELECT id FROM research_domain WHERE key='LS')), + ('LS9', 'Biotechnology and Biosystems Engineering', 'Biotechnology using all organisms, biotechnology for environment and food applications, applied plant and animal sciences, bioengineering and synthetic biology, biomass and biofuels, biohazards (LS9)', (SELECT id FROM research_domain WHERE key='LS')), + ('SH1', 'Individuals, Markets and Organisations', 'Economics, finance, management (SH1)', (SELECT id FROM research_domain WHERE key='SH')), + ('SH2', 'Institutions, Governance and Legal Systems', 'Political science, international relations, law (SH2)', (SELECT id FROM research_domain WHERE key='SH')), + ('SH3', 'The Social World and Its Diversity', 'Sociology, social psychology, social anthropology, education sciences, communication studies (SH3)', (SELECT id FROM research_domain WHERE key='SH')), + ('SH4', 'The Human Mind and Its Complexity', 'Cognitive science, psychology, linguistics, theoretical philosophy (SH4)', (SELECT id FROM research_domain WHERE key='SH')), + ('SH5', 'Cultures and Cultural Production', 'Literary studies, cultural studies, study of the arts, philosophy (SH5)', (SELECT id FROM research_domain WHERE key='SH')), + ('SH6', 'The Study of the Human Past', 'Archaeology and history (SH6)', (SELECT id FROM research_domain WHERE key='SH')), + ('SH7', 'Human Mobility, Environment, and Space', 'Human geography, demography, health, sustainability science, territorial planning, spatial analysis (SH7)', (SELECT id FROM research_domain WHERE key='SH')) +; + + +-- level 3 + +INSERT INTO research_domain (key,name,description,parent) +VALUES + ('PE1_1', 'Logic and foundations', 'Logic and foundations (PE1_1)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_2', 'Algebra', 'Algebra (PE1_2)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_3', 'Number theory', 'Number theory (PE1_3)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_4', 'Algebraic and complex geometry', 'Algebraic and complex geometry (PE1_4)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_5', 'Lie groups, Lie algebras', 'Lie groups, Lie algebras (PE1_5)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_6', 'Geometry and global analysis', 'Geometry and global analysis (PE1_6)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_7', 'Topology', 'Topology (PE1_7)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_8', 'Analysis', 'Analysis (PE1_8)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_9', 'Operator algebras and functional analysis', 'Operator algebras and functional analysis (PE1_9)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_10', 'ODE and dynamical systems', 'ODE and dynamical systems (PE1_10)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_11', 'Theoretical aspects of partial differential equations', 'Theoretical aspects of partial differential equations (PE1_11)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_12', 'Mathematical physics', 'Mathematical physics (PE1_12)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_13', 'Probability', 'Probability (PE1_13)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_14', 'Mathematical statistics', 'Mathematical statistics (PE1_14)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_15', 'Generic statistical methodology and modelling', 'Generic statistical methodology and modelling (PE1_15)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_16', 'Discrete mathematics and combinatorics', 'Discrete mathematics and combinatorics (PE1_16)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_17', 'Mathematical aspects of computer science', 'Mathematical aspects of computer science (PE1_17)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_18', 'Numerical analysis', 'Numerical analysis (PE1_18)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_19', 'Scientific computing and data processing', 'Scientific computing and data processing (PE1_19)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_20', 'Control theory, optimisation and operational research', 'Control theory, optimisation and operational research (PE1_20)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_21', 'Application of mathematics in sciences', 'Application of mathematics in sciences (PE1_21)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE1_22', 'Application of mathematics in industry and society', 'Application of mathematics in industry and society (PE1_22)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE2_1', 'Theory of fundamental interactions', 'Theory of fundamental interactions (PE2_1)', (SELECT id FROM research_domain WHERE key='PE1')), + ('PE2_2', 'Phenomenology of fundamental interactions', 'Phenomenology of fundamental interactions (PE2_2)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_3', 'Experimental particle physics with accelerators', 'Experimental particle physics with accelerators (PE2_3)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_4', 'Experimental particle physics without accelerators', 'Experimental particle physics without accelerators (PE2_4)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_5', 'Classical and quantum physics of gravitational interactions', 'Classical and quantum physics of gravitational interactions (PE2_5)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_6', 'Nuclear, hadron and heavy ion physics', 'Nuclear, hadron and heavy ion physics (PE2_6)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_7', 'Nuclear and particle astrophysics', 'Nuclear and particle astrophysics (PE2_7)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_8', 'Gas and plasma physics', 'Gas and plasma physics (PE2_8)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_9', 'Electromagnetism', 'Electromagnetism (PE2_9)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_10', 'Atomic, molecular physics', 'Atomic, molecular physics (PE2_10)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_11', 'Ultra-cold atoms and molecules', 'Ultra-cold atoms and molecules (PE2_11)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_12', 'Optics, non-linear optics and nano-optics', 'Optics, non-linear optics and nano-optics (PE2_12)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_13', 'Quantum optics and quantum information', 'Quantum optics and quantum information (PE2_13)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_14', 'Lasers, ultra-short lasers and laser physics', 'Lasers, ultra-short lasers and laser physics (PE2_14)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_15', 'Thermodynamics', 'Thermodynamics (PE2_15)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_16', 'Non-linear physics', 'Non-linear physics (PE2_16)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_17', 'Metrology and measurement', 'Metrology and measurement (PE2_17)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE2_18', 'Equilibrium and non-equilibrium statistical mechanics: steady states and dynamics', 'Equilibrium and non-equilibrium statistical mechanics: steady states and dynamics (PE2_18)', (SELECT id FROM research_domain WHERE key='PE2')), + ('PE3_1', 'Structure of solids, material growth and characterisation', 'Structure of solids, material growth and characterisation (PE3_1)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE3_2', 'Mechanical and acoustical properties of condensed matter, lattice dynamics', 'Mechanical and acoustical properties of condensed matter, lattice dynamics (PE3_2)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE3_3', 'Transport properties of condensed matter', 'Transport properties of condensed matter (PE3_3)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE3_4', 'Electronic properties of materials, surfaces, interfaces, nanostructures', 'Electronic properties of materials, surfaces, interfaces, nanostructures (PE3_4)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE3_5', 'Physical properties of semiconductors and insulators', 'Physical properties of semiconductors and insulators (PE3_5)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE3_6', 'Macroscopic quantum phenomena, e.g. superconductivity, superfluidity, quantum Hall effect', 'Macroscopic quantum phenomena, e.g. superconductivity, superfluidity, quantum Hall effect (PE3_6)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE3_7', 'Spintronics', 'Spintronics (PE3_7)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE3_8', 'Magnetism and strongly correlated systems', 'Magnetism and strongly correlated systems (PE3_8)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE3_9', 'Condensed matter – beam interactions (photons, electrons, etc.)', 'Condensed matter – beam interactions (photons, electrons, etc.) (PE3_9)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE3_10', 'Nanophysics, e.g. nanoelectronics, nanophotonics, nanomagnetism, nanoelectromechanics', 'Nanophysics, e.g. nanoelectronics, nanophotonics, nanomagnetism, nanoelectromechanics (PE3_10)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE3_11', 'Mesoscopic quantum physics and solid-state quantum technologies', 'Mesoscopic quantum physics and solid-state quantum technologies (PE3_11)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE3_12', 'Molecular electronics', 'Molecular electronics (PE3_12)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE3_13', 'Structure and dynamics of disordered systems, e.g. soft matter (gels, colloids, liquid crystals), granular matter, liquids, glasses, defects', 'Structure and dynamics of disordered systems, e.g. soft matter (gels, colloids, liquid crystals), granular matter, liquids, glasses, defects (PE3_13)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE3_14', 'Fluid dynamics (physics)', 'Fluid dynamics (physics) (PE3_14)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE3_15', 'Statistical physics: phase transitions, condensed matter systems, models of complex systems, interdisciplinary applications', 'Statistical physics: phase transitions, condensed matter systems, models of complex systems, interdisciplinary applications (PE3_15)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE3_16', 'Physics of biological systems', 'Physics of biological systems (PE3_16)', (SELECT id FROM research_domain WHERE key='PE3')), + ('PE4_1', 'Physical chemistry', 'Physical chemistry (PE4_1)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_2', 'Spectroscopic and spectrometric techniques', 'Spectroscopic and spectrometric techniques (PE4_2)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_3', 'Molecular architecture and Structure', 'Molecular architecture and Structure (PE4_3)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_4', 'Surface science and nanostructures', 'Surface science and nanostructures (PE4_4)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_5', 'Analytical chemistry', 'Analytical chemistry (PE4_5)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_6', 'Chemical physics', 'Chemical physics (PE4_6)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_7', 'Chemical instrumentation', 'Chemical instrumentation (PE4_7)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_8', 'Electrochemistry, electrodialysis, microfluidics, sensors', 'Electrochemistry, electrodialysis, microfluidics, sensors (PE4_8)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_9', 'Method development in chemistry', 'Method development in chemistry (PE4_9)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_10', 'Heterogeneous catalysis', 'Heterogeneous catalysis (PE4_10)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_11', 'Physical chemistry of biological systems', 'Physical chemistry of biological systems (PE4_11)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_12', 'Chemical reactions: mechanisms, dynamics, kinetics and catalytic reactions', 'Chemical reactions: mechanisms, dynamics, kinetics and catalytic reactions (PE4_12)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_13', 'Theoretical and computational chemistry', 'Theoretical and computational chemistry (PE4_13)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_14', 'Radiation and Nuclear chemistry', 'Radiation and Nuclear chemistry (PE4_14)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_15', 'Photochemistry', 'Photochemistry (PE4_15)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_16', 'Corrosion', 'Corrosion (PE4_16)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_17', 'Characterisation methods of materials', 'Characterisation methods of materials (PE4_17)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE4_18', 'Environment chemistry', 'Environment chemistry (PE4_18)', (SELECT id FROM research_domain WHERE key='PE4')), + ('PE5_1', 'Structural properties of materials', 'Structural properties of materials (PE5_1)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_2', 'Solid state materials chemistry', 'Solid state materials chemistry (PE5_2)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_3', 'Surface modification', 'Surface modification (PE5_3)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_4', 'Thin films', 'Thin films (PE5_4)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_5', 'Ionic liquids', 'Ionic liquids (PE5_5)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_6', 'New materials: oxides, alloys, composite, organic-inorganic hybrid, nanoparticles', 'New materials: oxides, alloys, composite, organic-inorganic hybrid, nanoparticles (PE5_6)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_7', 'Biomaterials synthesis', 'Biomaterials synthesis (PE5_7)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_8', 'Intelligent materials synthesis – self assembled materials', 'Intelligent materials synthesis – self assembled materials (PE5_8)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_9', 'Coordination chemistry', 'Coordination chemistry (PE5_9)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_10', 'Colloid chemistry', 'Colloid chemistry (PE5_10)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_11', 'Biological chemistry and chemical biology', 'Biological chemistry and chemical biology (PE5_11)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_12', 'Chemistry of condensed matter', 'Chemistry of condensed matter (PE5_12)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_13', 'Homogeneous catalysis', 'Homogeneous catalysis (PE5_13)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_14', 'Macromolecular chemistry', 'Macromolecular chemistry (PE5_14)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_15', 'Polymer chemistry', 'Polymer chemistry (PE5_15)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_16', 'Supramolecular chemistry', 'Supramolecular chemistry (PE5_16)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_17', 'Organic chemistry', 'Organic chemistry (PE5_17)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE5_18', 'Medicinal chemistry', 'Medicinal chemistry (PE5_18)', (SELECT id FROM research_domain WHERE key='PE5')), + ('PE6_1', 'Computer architecture, embedded systems, operating systems', 'Computer architecture, embedded systems, operating systems (PE6_1)', (SELECT id FROM research_domain WHERE key='PE6')), + ('PE6_2', 'Distributed systems, parallel computing, sensor networks, cyber-physical systems', 'Distributed systems, parallel computing, sensor networks, cyber-physical systems (PE6_2)', (SELECT id FROM research_domain WHERE key='PE6')), + ('PE6_3', 'Software engineering, programming languages and systems', 'Software engineering, programming languages and systems (PE6_3)', (SELECT id FROM research_domain WHERE key='PE6')), + ('PE6_4', 'Theoretical computer science, formal methods, automata', 'Theoretical computer science, formal methods, automata (PE6_4)', (SELECT id FROM research_domain WHERE key='PE6')), + ('PE6_5', 'Security, privacy, cryptology, quantum cryptography', 'Security, privacy, cryptology, quantum cryptography (PE6_5)', (SELECT id FROM research_domain WHERE key='PE6')), + ('PE6_6', 'Algorithms and complexity, distributed, parallel and network algorithms, algorithmic game theory', 'Algorithms and complexity, distributed, parallel and network algorithms, algorithmic game theory (PE6_6)', (SELECT id FROM research_domain WHERE key='PE6')), + ('PE6_7', 'Artificial intelligence, intelligent systems, natural language processing', 'Artificial intelligence, intelligent systems, natural language processing (PE6_7)', (SELECT id FROM research_domain WHERE key='PE6')), + ('PE6_8', 'Computer graphics, computer vision, multimedia, computer games', 'Computer graphics, computer vision, multimedia, computer games (PE6_8)', (SELECT id FROM research_domain WHERE key='PE6')), + ('PE6_9', 'Human computer interaction and interface, visualisation', 'Human computer interaction and interface, visualisation (PE6_9)', (SELECT id FROM research_domain WHERE key='PE6')), + ('PE6_10', 'Web and information systems, data management systems, information retrieval and digital libraries, data fusion', 'Web and information systems, data management systems, information retrieval and digital libraries, data fusion (PE6_10)', (SELECT id FROM research_domain WHERE key='PE6')), + ('PE6_11', 'Machine learning, statistical data processing and applications using signal processing ( e.g. speech,image,video)', 'Machine learning, statistical data processing and applications using signal processing ( e.g. speech,image,video) (PE6_11)', (SELECT id FROM research_domain WHERE key='PE6')), + ('PE6_12', 'Scientific computing, simulation and modelling tools', 'Scientific computing, simulation and modelling tools (PE6_12)', (SELECT id FROM research_domain WHERE key='PE6')), + ('PE6_13', 'Bioinformatics, bio-inspired computing, and natural computing', 'Bioinformatics, bio-inspired computing, and natural computing (PE6_13)', (SELECT id FROM research_domain WHERE key='PE6')), + ('PE6_14', 'Quantum computing (formal methods, algorithms and other computer science aspects)', 'Quantum computing (formal methods, algorithms and other computer science aspects) (PE6_14)', (SELECT id FROM research_domain WHERE key='PE6')), + ('PE7_1', 'Control engineering', 'Control engineering (PE7_1)', (SELECT id FROM research_domain WHERE key='PE7')), + ('PE7_2', 'Electrical engineering: power components and/or systems', 'Electrical engineering: power components and/or systems (PE7_2)', (SELECT id FROM research_domain WHERE key='PE7')), + ('PE7_3', 'Simulation engineering and modelling', 'Simulation engineering and modelling (PE7_3)', (SELECT id FROM research_domain WHERE key='PE7')), + ('PE7_4', '(Micro- and nano-) systems engineering', '(Micro- and nano-) systems engineering (PE7_4)', (SELECT id FROM research_domain WHERE key='PE7')), + ('PE7_5', '(Micro- and nano-) electronic, optoelectronic and photonic components', '(Micro- and nano-) electronic, optoelectronic and photonic components (PE7_5)', (SELECT id FROM research_domain WHERE key='PE7')), + ('PE7_6', 'Communication systems, wireless technology, high-frequency technology', 'Communication systems, wireless technology, high-frequency technology (PE7_6)', (SELECT id FROM research_domain WHERE key='PE7')), + ('PE7_7', 'Signal processing', 'Signal processing (PE7_7)', (SELECT id FROM research_domain WHERE key='PE7')), + ('PE7_8', 'Networks, e.g. communication networks and nodes, Internet of Things, sensor networks, networks of robots', 'Networks, e.g. communication networks and nodes, Internet of Things, sensor networks, networks of robots (PE7_8)', (SELECT id FROM research_domain WHERE key='PE7')), + ('PE7_9', 'Man-machine interfaces', 'Man-machine interfaces (PE7_9)', (SELECT id FROM research_domain WHERE key='PE7')), + ('PE7_10', 'Robotics', 'Robotics (PE7_10)', (SELECT id FROM research_domain WHERE key='PE7')), + ('PE7_11', 'Components and systems for applications (in e.g. medicine, biology, environment)', 'Components and systems for applications (in e.g. medicine, biology, environment) (PE7_11)', (SELECT id FROM research_domain WHERE key='PE7')), + ('PE7_12', 'Electrical energy production, distribution, applications', 'Electrical energy production, distribution, applications (PE7_12)', (SELECT id FROM research_domain WHERE key='PE7')), + ('PE8_1', 'Aerospace engineering', 'Aerospace engineering (PE8_1)', (SELECT id FROM research_domain WHERE key='PE8')), + ('PE8_2', 'Chemical engineering, technical chemistry', 'Chemical engineering, technical chemistry (PE8_2)', (SELECT id FROM research_domain WHERE key='PE8')), + ('PE8_3', 'Civil engineering, architecture, offshore construction, lightweight construction, geotechnics', 'Civil engineering, architecture, offshore construction, lightweight construction, geotechnics (PE8_3)', (SELECT id FROM research_domain WHERE key='PE8')), + ('PE8_4', 'Computational engineering', 'Computational engineering (PE8_4)', (SELECT id FROM research_domain WHERE key='PE8')), + ('PE8_5', 'Fluid mechanics', 'Fluid mechanics (PE8_5)', (SELECT id FROM research_domain WHERE key='PE8')), + ('PE8_6', 'Energy processes engineering', 'Energy processes engineering (PE8_6)', (SELECT id FROM research_domain WHERE key='PE8')), + ('PE8_7', 'Mechanical engineering', 'Mechanical engineering (PE8_7)', (SELECT id FROM research_domain WHERE key='PE8')), + ('PE8_8', 'Propulsion engineering, e.g. hydraulic, turbo, piston, hybrid engines', 'Propulsion engineering, e.g. hydraulic, turbo, piston, hybrid engines (PE8_8)', (SELECT id FROM research_domain WHERE key='PE8')), + ('PE8_9', 'Production technology, process engineering', 'Production technology, process engineering (PE8_9)', (SELECT id FROM research_domain WHERE key='PE8')), + ('PE8_10', 'Manufacturing engineering and industrial design', 'Manufacturing engineering and industrial design (PE8_10)', (SELECT id FROM research_domain WHERE key='PE8')), + ('PE8_11', 'Environmental engineering, e.g. sustainable design, waste and water treatment, recycling, regeneration or recovery of compounds, carbon capture & storage', 'Environmental engineering, e.g. sustainable design, waste and water treatment, recycling, regeneration or recovery of compounds, carbon capture & storage (PE8_11)', (SELECT id FROM research_domain WHERE key='PE8')), + ('PE8_12', 'Naval/marine engineering', 'Naval/marine engineering (PE8_12)', (SELECT id FROM research_domain WHERE key='PE8')), + ('PE8_13', 'Industrial bioengineering', 'Industrial bioengineering (PE8_13)', (SELECT id FROM research_domain WHERE key='PE8')), + ('PE8_14', 'Automotive and rail engineering; multi-/inter-modal transport engineering', 'Automotive and rail engineering; multi-/inter-modal transport engineering (PE8_14)', (SELECT id FROM research_domain WHERE key='PE8')), + ('PE9_1', 'Solar physics – the Sun and the heliosphere', 'Solar physics – the Sun and the heliosphere (PE9_1)', (SELECT id FROM research_domain WHERE key='PE9')), + ('PE9_2', 'Solar system science', 'Solar system science (PE9_2)', (SELECT id FROM research_domain WHERE key='PE9')), + ('PE9_3', 'Exoplanetary science, formation and characterization of extrasolar planets', 'Exoplanetary science, formation and characterization of extrasolar planets (PE9_3)', (SELECT id FROM research_domain WHERE key='PE9')), + ('PE9_4', 'Astrobiology', 'Astrobiology (PE9_4)', (SELECT id FROM research_domain WHERE key='PE9')), + ('PE9_5', 'Interstellar medium and star formation', 'Interstellar medium and star formation (PE9_5)', (SELECT id FROM research_domain WHERE key='PE9')), + ('PE9_6', 'Stars – stellar physics, stellar systems', 'Stars – stellar physics, stellar systems (PE9_6)', (SELECT id FROM research_domain WHERE key='PE9')), + ('PE9_7', 'The Milky Way', 'The Milky Way (PE9_7)', (SELECT id FROM research_domain WHERE key='PE9')), + ('PE9_8', 'Galaxies – formation, evolution, clusters', 'Galaxies – formation, evolution, clusters (PE9_8)', (SELECT id FROM research_domain WHERE key='PE9')), + ('PE9_9', 'Cosmology and large-scale structure, dark matter, dark energy', 'Cosmology and large-scale structure, dark matter, dark energy (PE9_9)', (SELECT id FROM research_domain WHERE key='PE9')), + ('PE9_10', 'Relativistic astrophysics and compact objects', 'Relativistic astrophysics and compact objects (PE9_10)', (SELECT id FROM research_domain WHERE key='PE9')), + ('PE9_11', 'Gravitational wave astronomy', 'Gravitational wave astronomy (PE9_11)', (SELECT id FROM research_domain WHERE key='PE9')), + ('PE9_12', 'High-energy and particle astronomy', 'High-energy and particle astronomy (PE9_12)', (SELECT id FROM research_domain WHERE key='PE9')), + ('PE9_13', 'Astronomical instrumentation and data, e.g. telescopes, detectors, techniques, archives, analyses', 'Astronomical instrumentation and data, e.g. telescopes, detectors, techniques, archives, analyses (PE9_13)', (SELECT id FROM research_domain WHERE key='PE9')), + ('PE10_1', 'Atmospheric chemistry, atmospheric composition, air pollution', 'Atmospheric chemistry, atmospheric composition, air pollution (PE10_1)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_2', 'Meteorology, atmospheric physics and dynamics', 'Meteorology, atmospheric physics and dynamics (PE10_2)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_3', 'Climatology and climate change', 'Climatology and climate change (PE10_3)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_4', 'Terrestrial ecology, land cover change', 'Terrestrial ecology, land cover change (PE10_4)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_5', 'Geology, tectonics, volcanology', 'Geology, tectonics, volcanology (PE10_5)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_6', 'Palaeoclimatology, palaeoecology', 'Palaeoclimatology, palaeoecology (PE10_6)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_7', 'Physics of earth’s interior, seismology, geodynamics', 'Physics of earth’s interior, seismology, geodynamics (PE10_7)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_8', 'Oceanography (physical, chemical, biological, geological)', 'Oceanography (physical, chemical, biological, geological) (PE10_8)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_9', 'Biogeochemistry, biogeochemical cycles, environmental chemistry', 'Biogeochemistry, biogeochemical cycles, environmental chemistry (PE10_9)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_10', 'Mineralogy, petrology, igneous petrology, metamorphic petrology', 'Mineralogy, petrology, igneous petrology, metamorphic petrology (PE10_10)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_11', 'Geochemistry, cosmochemistry, crystal chemistry, isotope geochemistry, thermodynamics', 'Geochemistry, cosmochemistry, crystal chemistry, isotope geochemistry, thermodynamics (PE10_11)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_12', 'Sedimentology, soil science, palaeontology, earth evolution', 'Sedimentology, soil science, palaeontology, earth evolution (PE10_12)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_13', 'Physical geography, geomorphology', 'Physical geography, geomorphology (PE10_13)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_14', 'Earth observations from space/remote sensing', 'Earth observations from space/remote sensing (PE10_14)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_15', 'Geomagnetism, palaeomagnetism', 'Geomagnetism, palaeomagnetism (PE10_15)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_16', 'Ozone, upper atmosphere, ionosphere', 'Ozone, upper atmosphere, ionosphere (PE10_16)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_17', 'Hydrology, hydrogeology, engineering and environmental geology, water and soil pollution', 'Hydrology, hydrogeology, engineering and environmental geology, water and soil pollution (PE10_17)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_18', 'Cryosphere, dynamics of snow and ice cover, sea ice, permafrosts and ice sheets', 'Cryosphere, dynamics of snow and ice cover, sea ice, permafrosts and ice sheets (PE10_18)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_19', 'Planetary geology and geophysics', 'Planetary geology and geophysics (PE10_19)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_20', 'Geohazards', 'Geohazards (PE10_20)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE10_21', 'Earth system modelling and interactions', 'Earth system modelling and interactions (PE10_21)', (SELECT id FROM research_domain WHERE key='PE10')), + ('PE11_1', 'Engineering of biomaterials, biomimetic, bioinspired and bio-enabled materials', 'Engineering of biomaterials, biomimetic, bioinspired and bio-enabled materials (PE11_1)', (SELECT id FROM research_domain WHERE key='PE11')), + ('PE11_2', 'Engineering of metals and alloys', 'Engineering of metals and alloys (PE11_2)', (SELECT id FROM research_domain WHERE key='PE11')), + ('PE11_3', 'Engineering of ceramics and glasses', 'Engineering of ceramics and glasses (PE11_3)', (SELECT id FROM research_domain WHERE key='PE11')), + ('PE11_4', 'Engineering of polymers and plastics', 'Engineering of polymers and plastics (PE11_4)', (SELECT id FROM research_domain WHERE key='PE11')), + ('PE11_5', 'Engineering of composites and hybrid materials', 'Engineering of composites and hybrid materials (PE11_5)', (SELECT id FROM research_domain WHERE key='PE11')), + ('PE11_6', 'Engineering of carbon materials', 'Engineering of carbon materials (PE11_6)', (SELECT id FROM research_domain WHERE key='PE11')), + ('PE11_7', 'Engineering of metal oxides', 'Engineering of metal oxides (PE11_7)', (SELECT id FROM research_domain WHERE key='PE11')), + ('PE11_8', 'Engineering of alternative established or emergent materials', 'Engineering of alternative established or emergent materials (PE11_8)', (SELECT id FROM research_domain WHERE key='PE11')), + ('PE11_9', 'Nanomaterials engineering, e.g. nanoparticles, nanoporous materials, 1D & 2D nanomaterials', 'Nanomaterials engineering, e.g. nanoparticles, nanoporous materials, 1D & 2D nanomaterials (PE11_9)', (SELECT id FROM research_domain WHERE key='PE11')), + ('PE11_10', 'Soft materials engineering, e.g. gels, foams, colloids', 'Soft materials engineering, e.g. gels, foams, colloids (PE11_10)', (SELECT id FROM research_domain WHERE key='PE11')), + ('PE11_11', 'Porous materials engineering, e.g. covalent-organic, metal-organic, porous aromatic frameworks', 'Porous materials engineering, e.g. covalent-organic, metal-organic, porous aromatic frameworks (PE11_11)', (SELECT id FROM research_domain WHERE key='PE11')), + ('PE11_12', 'Semi-conducting and magnetic materials engineering', 'Semi-conducting and magnetic materials engineering (PE11_12)', (SELECT id FROM research_domain WHERE key='PE11')), + ('PE11_13', 'Metamaterials engineering', 'Metamaterials engineering (PE11_13)', (SELECT id FROM research_domain WHERE key='PE11')), + ('PE11_14', 'Computational methods for materials engineering', 'Computational methods for materials engineering (PE11_14)', (SELECT id FROM research_domain WHERE key='PE11')), + ('LS1_1', 'Macromolecular complexes including interactions involving nucleic acids, proteins, lipids and carbohydrates', 'Macromolecular complexes including interactions involving nucleic acids, proteins, lipids and carbohydrates (LS1_1)', (SELECT id FROM research_domain WHERE key='LS1')), + ('LS1_2', 'Biochemistry', 'Biochemistry (LS1_2)', (SELECT id FROM research_domain WHERE key='LS1')), + ('LS1_3', 'DNA and RNA biology', 'DNA and RNA biology (LS1_3)', (SELECT id FROM research_domain WHERE key='LS1')), + ('LS1_4', 'Protein biology', 'Protein biology (LS1_4)', (SELECT id FROM research_domain WHERE key='LS1')), + ('LS1_5', 'Lipid biology', 'Lipid biology (LS1_5)', (SELECT id FROM research_domain WHERE key='LS1')), + ('LS1_6', 'Glycobiology', 'Glycobiology (LS1_6)', (SELECT id FROM research_domain WHERE key='LS1')), + ('LS1_7', 'Molecular biophysics, biomechanics, bioenergetics', 'Molecular biophysics, biomechanics, bioenergetics (LS1_7)', (SELECT id FROM research_domain WHERE key='LS1')), + ('LS1_8', 'Structural biology', 'Structural biology (LS1_8)', (SELECT id FROM research_domain WHERE key='LS1')), + ('LS1_9', 'Molecular mechanisms of signalling processes', 'Molecular mechanisms of signalling processes (LS1_9)', (SELECT id FROM research_domain WHERE key='LS1')), + ('LS1_10', 'Synthetic biology', 'Synthetic biology (LS1_10)', (SELECT id FROM research_domain WHERE key='LS1')), + ('LS1_11', 'Chemical biology', 'Chemical biology (LS1_11)', (SELECT id FROM research_domain WHERE key='LS1')), + ('LS1_12', 'Protein design', 'Protein design (LS1_12)', (SELECT id FROM research_domain WHERE key='LS1')), + ('LS1_13', 'Early translational research and drug design', 'Early translational research and drug design (LS1_13)', (SELECT id FROM research_domain WHERE key='LS1')), + ('LS1_14', 'Innovative methods and modelling in molecular, structural and synthetic biology', 'Innovative methods and modelling in molecular, structural and synthetic biology (LS1_14)', (SELECT id FROM research_domain WHERE key='LS1')), + ('LS2_1', 'Genetics', 'Genetics (LS2_1)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS2_2', 'Gene editing', 'Gene editing (LS2_2)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS2_3', 'Epigenetics', 'Epigenetics (LS2_3)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS2_4', 'Gene regulation', 'Gene regulation (LS2_4)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS2_5', 'Genomics', 'Genomics (LS2_5)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS2_6', 'Metagenomics', 'Metagenomics (LS2_6)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS2_7', 'Transcriptomics', 'Transcriptomics (LS2_7)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS2_8', 'Proteomics', 'Proteomics (LS2_8)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS2_9', 'Metabolomics', 'Metabolomics (LS2_9)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS2_10', 'Glycomics/Lipidomics', 'Glycomics/Lipidomics (LS2_10)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS2_11', 'Bioinformatics and computational biology', 'Bioinformatics and computational biology (LS2_11)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS2_12', 'Biostatistics', 'Biostatistics (LS2_12)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS2_13', 'Systems biology', 'Systems biology (LS2_13)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS2_14', 'Genetic diseases', 'Genetic diseases (LS2_14)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS2_15', 'Integrative biology for personalised medicine', 'Integrative biology for personalised medicine (LS2_15)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS2_16', 'Innovative methods and modelling in integrative biology', 'Innovative methods and modelling in integrative biology (LS2_16)', (SELECT id FROM research_domain WHERE key='LS2')), + ('LS3_1', 'Cell cycle, cell division and growth', 'Cell cycle, cell division and growth (LS3_1)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_2', 'Cell senescence, cell death, autophagy, cell ageing', 'Cell senescence, cell death, autophagy, cell ageing (LS3_2)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_3', 'Cell behaviour, including control of cell shape, cell migration', 'Cell behaviour, including control of cell shape, cell migration (LS3_3)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_4', 'Cell junctions, cell adhesion, the extracellular matrix, cell communication', 'Cell junctions, cell adhesion, the extracellular matrix, cell communication (LS3_4)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_5', 'Cell signalling and signal transduction, exosome biology', 'Cell signalling and signal transduction, exosome biology (LS3_5)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_6', 'Organelle biology and trafficking', 'Organelle biology and trafficking (LS3_6)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_7', 'Mechanobiology of cells, tissues and organs', 'Mechanobiology of cells, tissues and organs (LS3_7)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_8', 'Embryogenesis, pattern formation, morphogenesis', 'Embryogenesis, pattern formation, morphogenesis (LS3_8)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_9', 'Cell differentiation, formation of tissues and organs', 'Cell differentiation, formation of tissues and organs (LS3_9)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_10', 'Developmental genetics', 'Developmental genetics (LS3_10)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_11', 'Evolution of developmental strategies', 'Evolution of developmental strategies (LS3_11)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_12', 'Organoids', 'Organoids (LS3_12)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_13', 'Stem cells', 'Stem cells (LS3_13)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_14', 'Regeneration', 'Regeneration (LS3_14)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_15', 'Development of cell-based therapeutic approaches for tissue regeneration', 'Development of cell-based therapeutic approaches for tissue regeneration (LS3_15)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_16', 'Functional imaging of cells and tissues', 'Functional imaging of cells and tissues (LS3_16)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS3_17', 'Theoretical modelling in cellular, developmental and regenerative biology', 'Theoretical modelling in cellular, developmental and regenerative biology (LS3_17)', (SELECT id FROM research_domain WHERE key='LS3')), + ('LS4_1', 'Organ and tissue physiology and pathophysiology', 'Organ and tissue physiology and pathophysiology (LS4_1)', (SELECT id FROM research_domain WHERE key='LS4')), + ('LS4_2', 'Comparative physiology', 'Comparative physiology (LS4_2)', (SELECT id FROM research_domain WHERE key='LS4')), + ('LS4_3', 'Physiology of ageing', 'Physiology of ageing (LS4_3)', (SELECT id FROM research_domain WHERE key='LS4')), + ('LS4_4', 'Endocrinology', 'Endocrinology (LS4_4)', (SELECT id FROM research_domain WHERE key='LS4')), + ('LS4_5', 'Non-hormonal mechanisms of inter-organ and tissue communication', 'Non-hormonal mechanisms of inter-organ and tissue communication (LS4_5)', (SELECT id FROM research_domain WHERE key='LS4')), + ('LS4_6', 'Microbiome and host physiology', 'Microbiome and host physiology (LS4_6)', (SELECT id FROM research_domain WHERE key='LS4')), + ('LS4_7', 'Nutrition and exercise physiology', 'Nutrition and exercise physiology (LS4_7)', (SELECT id FROM research_domain WHERE key='LS4')), + ('LS4_8', 'Impact of stress (including environmental stress) on physiology', 'Impact of stress (including environmental stress) on physiology (LS4_8)', (SELECT id FROM research_domain WHERE key='LS4')), + ('LS4_9', 'Metabolism and metabolic disorders, including diabetes and obesity', 'Metabolism and metabolic disorders, including diabetes and obesity (LS4_9)', (SELECT id FROM research_domain WHERE key='LS4')), + ('LS4_10', 'The cardiovascular system and cardiovascular diseases', 'The cardiovascular system and cardiovascular diseases (LS4_10)', (SELECT id FROM research_domain WHERE key='LS4')), + ('LS4_11', 'Haematopoiesis and blood diseases', 'Haematopoiesis and blood diseases (LS4_11)', (SELECT id FROM research_domain WHERE key='LS4')), + ('LS4_12', 'Cancer', 'Cancer (LS4_12)', (SELECT id FROM research_domain WHERE key='LS4')), + ('LS4_13', 'Other non-communicable diseases (except disorders of the nervous system and immunity-related diseases)', 'Other non-communicable diseases (except disorders of the nervous system and immunity-related diseases) (LS4_13)', (SELECT id FROM research_domain WHERE key='LS4')), + ('LS5_1', 'Neuronal cells', 'Neuronal cells (LS5_1)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_2', 'Glial cells and neuronal-glial communication', 'Glial cells and neuronal-glial communication (LS5_2)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_3', 'Neural development and related disorders', 'Neural development and related disorders (LS5_3)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_4', 'Neural stem cells', 'Neural stem cells (LS5_4)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_5', 'Neural networks and plasticity', 'Neural networks and plasticity (LS5_5)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_6', 'Neurovascular biology and blood-brain barrier', 'Neurovascular biology and blood-brain barrier (LS5_6)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_7', 'Sensory systems, sensation and perception, including pain', 'Sensory systems, sensation and perception, including pain (LS5_7)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_8', 'Neural basis of behaviour', 'Neural basis of behaviour (LS5_8)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_9', 'Neural basis of cognition', 'Neural basis of cognition (LS5_9)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_10', 'Ageing of the nervous system', 'Ageing of the nervous system (LS5_10)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_11', 'Neurological and neurodegenerative disorders', 'Neurological and neurodegenerative disorders (LS5_11)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_12', 'Mental disorders', 'Mental disorders (LS5_12)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_13', 'Nervous system injuries and trauma, stroke', 'Nervous system injuries and trauma, stroke (LS5_13)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_14', 'Repair and regeneration of the nervous system', 'Repair and regeneration of the nervous system (LS5_14)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_15', 'Neuroimmunology, neuroinflammation', 'Neuroimmunology, neuroinflammation (LS5_15)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_16', 'Systems and computational neuroscience', 'Systems and computational neuroscience (LS5_16)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_17', 'Imaging in neuroscience', 'Imaging in neuroscience (LS5_17)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS5_18', 'Innovative methods and tools for neuroscience', 'Innovative methods and tools for neuroscience (LS5_18)', (SELECT id FROM research_domain WHERE key='LS5')), + ('LS6_1', 'Innate immunity', 'Innate immunity (LS6_1)', (SELECT id FROM research_domain WHERE key='LS6')), + ('LS6_2', 'Adaptive immunity', 'Adaptive immunity (LS6_2)', (SELECT id FROM research_domain WHERE key='LS6')), + ('LS6_3', 'Regulation of the immune response', 'Regulation of the immune response (LS6_3)', (SELECT id FROM research_domain WHERE key='LS6')), + ('LS6_4', 'Immune-related diseases', 'Immune-related diseases (LS6_4)', (SELECT id FROM research_domain WHERE key='LS6')), + ('LS6_5', 'Biology of pathogens (e.g. bacteria, viruses, parasites, fungi)', 'Biology of pathogens (e.g. bacteria, viruses, parasites, fungi) (LS6_5)', (SELECT id FROM research_domain WHERE key='LS6')), + ('LS6_6', 'Infectious diseases', 'Infectious diseases (LS6_6)', (SELECT id FROM research_domain WHERE key='LS6')), + ('LS6_7', 'Mechanisms of infection', 'Mechanisms of infection (LS6_7)', (SELECT id FROM research_domain WHERE key='LS6')), + ('LS6_8', 'Biological basis of prevention and treatment of infection', 'Biological basis of prevention and treatment of infection (LS6_8)', (SELECT id FROM research_domain WHERE key='LS6')), + ('LS6_9', 'Antimicrobials, antimicrobial resistance', 'Antimicrobials, antimicrobial resistance (LS6_9)', (SELECT id FROM research_domain WHERE key='LS6')), + ('LS6_10', 'Vaccine development', 'Vaccine development (LS6_10)', (SELECT id FROM research_domain WHERE key='LS6')), + ('LS6_11', 'Innovative immunological tools and approaches, including therapies', 'Innovative immunological tools and approaches, including therapies (LS6_11)', (SELECT id FROM research_domain WHERE key='LS6')), + ('LS7_1', 'Medical imaging for prevention, diagnosis and monitoring of diseases', 'Medical imaging for prevention, diagnosis and monitoring of diseases (LS7_1)', (SELECT id FROM research_domain WHERE key='LS7')), + ('LS7_2', 'Medical technologies and tools (including genetic tools and biomarkers) for prevention, diagnosis, monitoring and treatment of diseases', 'Medical technologies and tools (including genetic tools and biomarkers) for prevention, diagnosis, monitoring and treatment of diseases (LS7_2)', (SELECT id FROM research_domain WHERE key='LS7')), + ('LS7_3', 'Nanomedicine', 'Nanomedicine (LS7_3)', (SELECT id FROM research_domain WHERE key='LS7')), + ('LS7_4', 'Regenerative medicine', 'Regenerative medicine (LS7_4)', (SELECT id FROM research_domain WHERE key='LS7')), + ('LS7_5', 'Applied gene, cell and immune therapies', 'Applied gene, cell and immune therapies (LS7_5)', (SELECT id FROM research_domain WHERE key='LS7')), + ('LS7_6', 'Other medical therapeutic interventions, including transplantation', 'Other medical therapeutic interventions, including transplantation (LS7_6)', (SELECT id FROM research_domain WHERE key='LS7')), + ('LS7_7', 'Pharmacology and toxicology', 'Pharmacology and toxicology (LS7_7)', (SELECT id FROM research_domain WHERE key='LS7')), + ('LS7_8', 'Effectiveness of interventions, including resistance to therapies', 'Effectiveness of interventions, including resistance to therapies (LS7_8)', (SELECT id FROM research_domain WHERE key='LS7')), + ('LS7_9', 'Public health and epidemiology', 'Public health and epidemiology (LS7_9)', (SELECT id FROM research_domain WHERE key='LS7')), + ('LS7_10', 'Preventative and prognostic medicine', 'Preventative and prognostic medicine (LS7_10)', (SELECT id FROM research_domain WHERE key='LS7')), + ('LS7_11', 'Environmental health, occupational medicine', 'Environmental health, occupational medicine (LS7_11)', (SELECT id FROM research_domain WHERE key='LS7')), + ('LS7_12', 'Health care, including care for the ageing population', 'Health care, including care for the ageing population (LS7_12)', (SELECT id FROM research_domain WHERE key='LS7')), + ('LS7_13', 'Palliative medicine', 'Palliative medicine (LS7_13)', (SELECT id FROM research_domain WHERE key='LS7')), + ('LS7_14', 'Digital medicine, e-medicine, medical applications of artificial intelligence', 'Digital medicine, e-medicine, medical applications of artificial intelligence (LS7_14)', (SELECT id FROM research_domain WHERE key='LS7')), + ('LS7_15', 'Medical ethics', 'Medical ethics (LS7_15)', (SELECT id FROM research_domain WHERE key='LS7')), + ('LS8_1', 'Ecosystem and community ecology, macroecology', 'Ecosystem and community ecology, macroecology (LS8_1)', (SELECT id FROM research_domain WHERE key='LS8')), + ('LS8_2', 'Biodiversity', 'Biodiversity (LS8_2)', (SELECT id FROM research_domain WHERE key='LS8')), + ('LS8_3', 'Conservation biology', 'Conservation biology (LS8_3)', (SELECT id FROM research_domain WHERE key='LS8')), + ('LS8_4', 'Population biology, population dynamics, population genetics', 'Population biology, population dynamics, population genetics (LS8_4)', (SELECT id FROM research_domain WHERE key='LS8')), + ('LS8_5', 'Biological aspects of environmental change, including climate change', 'Biological aspects of environmental change, including climate change (LS8_5)', (SELECT id FROM research_domain WHERE key='LS8')), + ('LS8_6', 'Evolutionary ecology', 'Evolutionary ecology (LS8_6)', (SELECT id FROM research_domain WHERE key='LS8')), + ('LS8_7', 'Evolutionary genetics', 'Evolutionary genetics (LS8_7)', (SELECT id FROM research_domain WHERE key='LS8')), + ('LS8_8', 'Phylogenetics, systematics, comparative biology', 'Phylogenetics, systematics, comparative biology (LS8_8)', (SELECT id FROM research_domain WHERE key='LS8')), + ('LS8_9', 'Macroevolution and paleobiology', 'Macroevolution and paleobiology (LS8_9)', (SELECT id FROM research_domain WHERE key='LS8')), + ('LS8_10', 'Ecology and evolution of species interactions', 'Ecology and evolution of species interactions (LS8_10)', (SELECT id FROM research_domain WHERE key='LS8')), + ('LS8_11', 'Behavioural ecology and evolution', 'Behavioural ecology and evolution (LS8_11)', (SELECT id FROM research_domain WHERE key='LS8')), + ('LS8_12', 'Microbial ecology and evolution', 'Microbial ecology and evolution (LS8_12)', (SELECT id FROM research_domain WHERE key='LS8')), + ('LS8_13', 'Marine biology and ecology', 'Marine biology and ecology (LS8_13)', (SELECT id FROM research_domain WHERE key='LS8')), + ('LS8_14', 'Ecophysiology, from organisms to ecosystems', 'Ecophysiology, from organisms to ecosystems (LS8_14)', (SELECT id FROM research_domain WHERE key='LS8')), + ('LS8_15', 'Theoretical developments and modelling in environmental biology, ecology, and evolution', 'Theoretical developments and modelling in environmental biology, ecology, and evolution (LS8_15)', (SELECT id FROM research_domain WHERE key='LS8')), + ('LS9_1', 'Bioengineering for synthetic and chemical biology', 'Bioengineering for synthetic and chemical biology (LS9_1)', (SELECT id FROM research_domain WHERE key='LS9')), + ('LS9_2', 'Applied genetics, gene editing and transgenic organisms', 'Applied genetics, gene editing and transgenic organisms (LS9_2)', (SELECT id FROM research_domain WHERE key='LS9')), + ('LS9_3', 'Bioengineering of cells, tissues, organs and organisms', 'Bioengineering of cells, tissues, organs and organisms (LS9_3)', (SELECT id FROM research_domain WHERE key='LS9')), + ('LS9_4', 'Microbial biotechnology and bioengineering', 'Microbial biotechnology and bioengineering (LS9_4)', (SELECT id FROM research_domain WHERE key='LS9')), + ('LS9_5', 'Food biotechnology and bioengineering', 'Food biotechnology and bioengineering (LS9_5)', (SELECT id FROM research_domain WHERE key='LS9')), + ('LS9_6', 'Marine biotechnology and bioengineering', 'Marine biotechnology and bioengineering (LS9_6)', (SELECT id FROM research_domain WHERE key='LS9')), + ('LS9_7', 'Environmental biotechnology and bioengineering', 'Environmental biotechnology and bioengineering (LS9_7)', (SELECT id FROM research_domain WHERE key='LS9')), + ('LS9_8', 'Applied plant sciences, plant breeding, agroecology and soil biology', 'Applied plant sciences, plant breeding, agroecology and soil biology (LS9_8)', (SELECT id FROM research_domain WHERE key='LS9')), + ('LS9_9', 'Plant pathology and pest resistance', 'Plant pathology and pest resistance (LS9_9)', (SELECT id FROM research_domain WHERE key='LS9')), + ('LS9_10', 'Veterinary and applied animal sciences', 'Veterinary and applied animal sciences (LS9_10)', (SELECT id FROM research_domain WHERE key='LS9')), + ('LS9_11', 'Biomass production and utilisation, biofuels', 'Biomass production and utilisation, biofuels (LS9_11)', (SELECT id FROM research_domain WHERE key='LS9')), + ('LS9_12', 'Ecotoxicology, biohazards and biosafety', 'Ecotoxicology, biohazards and biosafety (LS9_12)', (SELECT id FROM research_domain WHERE key='LS9')), + ('SH1_1', 'Macroeconomics; monetary economics; economic growth', 'Macroeconomics; monetary economics; economic growth (SH1_1)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH1_2', 'International trade; international management; international business; spatial economics', 'International trade; international management; international business; spatial economics (SH1_2)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH1_3', 'Development economics; structural change; political economy of development', 'Development economics; structural change; political economy of development (SH1_3)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH1_4', 'Finance; asset pricing; international finance; market microstructure', 'Finance; asset pricing; international finance; market microstructure (SH1_4)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH1_5', 'Corporate finance; banking and financial intermediation; accounting; auditing; insurance', 'Corporate finance; banking and financial intermediation; accounting; auditing; insurance (SH1_5)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH1_6', 'Econometrics; operations research', 'Econometrics; operations research (SH1_6)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH1_7', 'Behavioural economics; experimental economics; neuro-economics', 'Behavioural economics; experimental economics; neuro-economics (SH1_7)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH1_8', 'Microeconomic theory; game theory; decision theory', 'Microeconomic theory; game theory; decision theory (SH1_8)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH1_9', 'Industrial organisation; entrepreneurship; R&D and innovation', 'Industrial organisation; entrepreneurship; R&D and innovation (SH1_9)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH1_10', 'Management; strategy; organisational behaviour', 'Management; strategy; organisational behaviour (SH1_10)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH1_11', 'Human resource management; operations management, marketing', 'Human resource management; operations management, marketing (SH1_11)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH1_12', 'Environmental economics; resource and energy economics; agricultural economics', 'Environmental economics; resource and energy economics; agricultural economics (SH1_12)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH1_13', 'Labour and demographic economics', 'Labour and demographic economics (SH1_13)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH1_14', 'Health economics; economics of education', 'Health economics; economics of education (SH1_14)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH1_15', 'Public economics; political economics; law and economics', 'Public economics; political economics; law and economics (SH1_15)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH1_16', 'Historical economics; quantitative economic history; institutional economics; economic systems', 'Historical economics; quantitative economic history; institutional economics; economic systems (SH1_16)', (SELECT id FROM research_domain WHERE key='SH1')), + ('SH2_1', 'Political systems, governance', 'Political systems, governance (SH2_1)', (SELECT id FROM research_domain WHERE key='SH2')), + ('SH2_2', 'Democratisation and social movements', 'Democratisation and social movements (SH2_2)', (SELECT id FROM research_domain WHERE key='SH2')), + ('SH2_3', 'Conflict resolution, war, peace building, international law', 'Conflict resolution, war, peace building, international law (SH2_3)', (SELECT id FROM research_domain WHERE key='SH2')), + ('SH2_4', 'Legal studies, constitutions, human rights, comparative law', 'Legal studies, constitutions, human rights, comparative law (SH2_4)', (SELECT id FROM research_domain WHERE key='SH2')), + ('SH2_5', 'International relations, global and transnational governance', 'International relations, global and transnational governance (SH2_5)', (SELECT id FROM research_domain WHERE key='SH2')), + ('SH2_6', 'Humanitarian assistance and development', 'Humanitarian assistance and development (SH2_6)', (SELECT id FROM research_domain WHERE key='SH2')), + ('SH2_7', 'Political and legal philosophy', 'Political and legal philosophy (SH2_7)', (SELECT id FROM research_domain WHERE key='SH2')), + ('SH2_8', 'Big data in political and legal studies', 'Big data in political and legal studies (SH2_8)', (SELECT id FROM research_domain WHERE key='SH2')), + ('SH3_1', 'Social structure, social mobility, social innovation', 'Social structure, social mobility, social innovation (SH3_1)', (SELECT id FROM research_domain WHERE key='SH3')), + ('SH3_2', 'Inequalities, discrimination, prejudice', 'Inequalities, discrimination, prejudice (SH3_2)', (SELECT id FROM research_domain WHERE key='SH3')), + ('SH3_3', 'Aggression and violence, antisocial behaviour, crime', 'Aggression and violence, antisocial behaviour, crime (SH3_3)', (SELECT id FROM research_domain WHERE key='SH3')), + ('SH3_4', 'Social integration, exclusion, prosocial behaviour', 'Social integration, exclusion, prosocial behaviour (SH3_4)', (SELECT id FROM research_domain WHERE key='SH3')), + ('SH3_5', 'Attitudes and beliefs', 'Attitudes and beliefs (SH3_5)', (SELECT id FROM research_domain WHERE key='SH3')), + ('SH3_6', 'Social influence; power and group behaviour', 'Social influence; power and group behaviour (SH3_6)', (SELECT id FROM research_domain WHERE key='SH3')), + ('SH3_7', 'Kinship; diversity and identities, gender, interethnic relations', 'Kinship; diversity and identities, gender, interethnic relations (SH3_7)', (SELECT id FROM research_domain WHERE key='SH3')), + ('SH3_8', 'Social policies, welfare, work and employment', 'Social policies, welfare, work and employment (SH3_8)', (SELECT id FROM research_domain WHERE key='SH3')), + ('SH3_9', 'Poverty and poverty alleviation', 'Poverty and poverty alleviation (SH3_9)', (SELECT id FROM research_domain WHERE key='SH3')), + ('SH3_10', 'Religious studies, ritual; symbolic representation', 'Religious studies, ritual; symbolic representation (SH3_10)', (SELECT id FROM research_domain WHERE key='SH3')), + ('SH3_11', 'Social aspects of teaching and learning, curriculum studies, education and educational policies', 'Social aspects of teaching and learning, curriculum studies, education and educational policies (SH3_11)', (SELECT id FROM research_domain WHERE key='SH3')), + ('SH3_12', 'Communication and information, networks, media', 'Communication and information, networks, media (SH3_12)', (SELECT id FROM research_domain WHERE key='SH3')), + ('SH3_13', 'Digital social research', 'Digital social research (SH3_13)', (SELECT id FROM research_domain WHERE key='SH3')), + ('SH3_14', 'Social studies of science and technology', 'Social studies of science and technology (SH3_14)', (SELECT id FROM research_domain WHERE key='SH3')), + ('SH4_1', 'Cognitive basis of human development and education, developmental disorders; comparative cognition', 'Cognitive basis of human development and education, developmental disorders; comparative cognition (SH4_1)', (SELECT id FROM research_domain WHERE key='SH4')), + ('SH4_2', 'Personality and social cognition; emotion', 'Personality and social cognition; emotion (SH4_2)', (SELECT id FROM research_domain WHERE key='SH4')), + ('SH4_3', 'Clinical and health psychology', 'Clinical and health psychology (SH4_3)', (SELECT id FROM research_domain WHERE key='SH4')), + ('SH4_4', 'Neuropsychology', 'Neuropsychology (SH4_4)', (SELECT id FROM research_domain WHERE key='SH4')), + ('SH4_5', 'Attention, perception, action, consciousness', 'Attention, perception, action, consciousness (SH4_5)', (SELECT id FROM research_domain WHERE key='SH4')), + ('SH4_6', 'Learning, memory; cognition in ageing', 'Learning, memory; cognition in ageing (SH4_6)', (SELECT id FROM research_domain WHERE key='SH4')), + ('SH4_7', 'Reasoning, decision-making; intelligence', 'Reasoning, decision-making; intelligence (SH4_7)', (SELECT id FROM research_domain WHERE key='SH4')), + ('SH4_8', 'Language learning and processing (first and second languages)', 'Language learning and processing (first and second languages) (SH4_8)', (SELECT id FROM research_domain WHERE key='SH4')), + ('SH4_9', 'Theoretical linguistics; computational linguistics', 'Theoretical linguistics; computational linguistics (SH4_9)', (SELECT id FROM research_domain WHERE key='SH4')), + ('SH4_10', 'Language typology; historical linguistics', 'Language typology; historical linguistics (SH4_10)', (SELECT id FROM research_domain WHERE key='SH4')), + ('SH4_11', 'Pragmatics, sociolinguistics, linguistic anthropology, discourse analysis', 'Pragmatics, sociolinguistics, linguistic anthropology, discourse analysis (SH4_11)', (SELECT id FROM research_domain WHERE key='SH4')), + ('SH4_12', 'Philosophy of mind, philosophy of language', 'Philosophy of mind, philosophy of language (SH4_12)', (SELECT id FROM research_domain WHERE key='SH4')), + ('SH4_13', 'Philosophy of science, epistemology, logic', 'Philosophy of science, epistemology, logic (SH4_13)', (SELECT id FROM research_domain WHERE key='SH4')), + ('SH5_1', 'Classics, ancient literature and art', 'Classics, ancient literature and art (SH5_1)', (SELECT id FROM research_domain WHERE key='SH5')), + ('SH5_2', 'Theory and history of literature, comparative literature', 'Theory and history of literature, comparative literature (SH5_2)', (SELECT id FROM research_domain WHERE key='SH5')), + ('SH5_3', 'Philology; text and image studies', 'Philology; text and image studies (SH5_3)', (SELECT id FROM research_domain WHERE key='SH5')), + ('SH5_4', 'Visual and performing arts, film, design and architecture', 'Visual and performing arts, film, design and architecture (SH5_4)', (SELECT id FROM research_domain WHERE key='SH5')), + ('SH5_5', 'Music and musicology; history of music', 'Music and musicology; history of music (SH5_5)', (SELECT id FROM research_domain WHERE key='SH5')), + ('SH5_6', 'History of art and architecture, arts-based research', 'History of art and architecture, arts-based research (SH5_6)', (SELECT id FROM research_domain WHERE key='SH5')), + ('SH5_7', 'Museums, exhibitions, conservation and restoration', 'Museums, exhibitions, conservation and restoration (SH5_7)', (SELECT id FROM research_domain WHERE key='SH5')), + ('SH5_8', 'Cultural studies, cultural identities and memories, cultural heritage', 'Cultural studies, cultural identities and memories, cultural heritage (SH5_8)', (SELECT id FROM research_domain WHERE key='SH5')), + ('SH5_9', 'Metaphysics, philosophical anthropology; aesthetics', 'Metaphysics, philosophical anthropology; aesthetics (SH5_9)', (SELECT id FROM research_domain WHERE key='SH5')), + ('SH5_10', 'Ethics and its applications; social philosophy', 'Ethics and its applications; social philosophy (SH5_10)', (SELECT id FROM research_domain WHERE key='SH5')), + ('SH5_11', 'History of philosophy', 'History of philosophy (SH5_11)', (SELECT id FROM research_domain WHERE key='SH5')), + ('SH5_12', 'Computational modelling and digitisation in the cultural sphere', 'Computational modelling and digitisation in the cultural sphere (SH5_12)', (SELECT id FROM research_domain WHERE key='SH5')), + ('SH6_1', 'Historiography, theory and methods in history, including the analysis of digital data', 'Historiography, theory and methods in history, including the analysis of digital data (SH6_1)', (SELECT id FROM research_domain WHERE key='SH6')), + ('SH6_2', 'Classical archaeology, history of archaeology, social archaeology', 'Classical archaeology, history of archaeology, social archaeology (SH6_2)', (SELECT id FROM research_domain WHERE key='SH6')), + ('SH6_3', 'General archaeology, archaeometry, landscape archaeology', 'General archaeology, archaeometry, landscape archaeology (SH6_3)', (SELECT id FROM research_domain WHERE key='SH6')), + ('SH6_4', 'Prehistory, palaeoanthropology, palaeodemography, protohistory, bioarchaeology', 'Prehistory, palaeoanthropology, palaeodemography, protohistory, bioarchaeology (SH6_4)', (SELECT id FROM research_domain WHERE key='SH6')), + ('SH6_5', 'Palaeography and codicology', 'Palaeography and codicology (SH6_5)', (SELECT id FROM research_domain WHERE key='SH6')), + ('SH6_6', 'Ancient history', 'Ancient history (SH6_6)', (SELECT id FROM research_domain WHERE key='SH6')), + ('SH6_7', 'Medieval history', 'Medieval history (SH6_7)', (SELECT id FROM research_domain WHERE key='SH6')), + ('SH6_8', 'Early modern history', 'Early modern history (SH6_8)', (SELECT id FROM research_domain WHERE key='SH6')), + ('SH6_9', 'Modern and contemporary history', 'Modern and contemporary history (SH6_9)', (SELECT id FROM research_domain WHERE key='SH6')), + ('SH6_10', 'Colonial and post-colonial history', 'Colonial and post-colonial history (SH6_10)', (SELECT id FROM research_domain WHERE key='SH6')), + ('SH6_11', 'Global history, transnational history, comparative history, entangled histories', 'Global history, transnational history, comparative history, entangled histories (SH6_11)', (SELECT id FROM research_domain WHERE key='SH6')), + ('SH6_12', 'Social and economic history', 'Social and economic history (SH6_12)', (SELECT id FROM research_domain WHERE key='SH6')), + ('SH6_13', 'Gender history, cultural history, history of collective identities and memories, history of religions', 'Gender history, cultural history, history of collective identities and memories, history of religions (SH6_13)', (SELECT id FROM research_domain WHERE key='SH6')), + ('SH6_14', 'History of ideas, intellectual history, history of economic thought', 'History of ideas, intellectual history, history of economic thought (SH6_14)', (SELECT id FROM research_domain WHERE key='SH6')), + ('SH6_15', 'History of science, medicine and technologies', 'History of science, medicine and technologies (SH6_15)', (SELECT id FROM research_domain WHERE key='SH6')), + ('SH7_1', 'Human, economic and social geography', 'Human, economic and social geography (SH7_1)', (SELECT id FROM research_domain WHERE key='SH7')), + ('SH7_2', 'Migration', 'Migration (SH7_2)', (SELECT id FROM research_domain WHERE key='SH7')), + ('SH7_3', 'Population dynamics: households, family and fertility', 'Population dynamics: households, family and fertility (SH7_3)', (SELECT id FROM research_domain WHERE key='SH7')), + ('SH7_4', 'Social aspects of health, ageing and society', 'Social aspects of health, ageing and society (SH7_4)', (SELECT id FROM research_domain WHERE key='SH7')), + ('SH7_5', 'Sustainability sciences, environment and resources', 'Sustainability sciences, environment and resources (SH7_5)', (SELECT id FROM research_domain WHERE key='SH7')), + ('SH7_6', 'Environmental and climate change, societal impact and policy', 'Environmental and climate change, societal impact and policy (SH7_6)', (SELECT id FROM research_domain WHERE key='SH7')), + ('SH7_7', 'Cities; urban, regional and rural studies', 'Cities; urban, regional and rural studies (SH7_7)', (SELECT id FROM research_domain WHERE key='SH7')), + ('SH7_8', 'Land use and planning', 'Land use and planning (SH7_8)', (SELECT id FROM research_domain WHERE key='SH7')), + ('SH7_9', 'Energy, transportation and mobility', 'Energy, transportation and mobility (SH7_9)', (SELECT id FROM research_domain WHERE key='SH7')), + ('SH7_10', 'GIS, spatial analysis; big data in geographical studies', 'GIS, spatial analysis; big data in geographical studies (SH7_10)', (SELECT id FROM research_domain WHERE key='SH7')) +; diff --git a/database/011-create-mention-table.sql b/database/011-create-mention-table.sql new file mode 100644 index 0000000..07d678d --- /dev/null +++ b/database/011-create-mention-table.sql @@ -0,0 +1,169 @@ +-- SPDX-FileCopyrightText: 2021 - 2024 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2021 - 2024 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2022 dv4all +-- SPDX-FileCopyrightText: 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +-- SPDX-FileCopyrightText: 2025 Paula Stock (GFZ) +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE TYPE mention_type AS ENUM ( + 'blogPost', + 'book', + 'bookSection', + 'computerProgram', + 'conferencePaper', + 'dataset', + 'interview', + 'highlight', + 'journalArticle', + 'magazineArticle', + 'newspaperArticle', + 'poster', + 'presentation', + 'report', + 'thesis', + 'videoRecording', + 'webpage', + 'workshop', + 'other' +); + +CREATE TABLE mention ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + doi CITEXT UNIQUE CHECK (doi ~ '^10(\.\w+)+/\S+$' AND LENGTH(doi) <= 255), + doi_registration_date TIMESTAMPTZ, + openalex_id CITEXT UNIQUE CHECK (openalex_id ~ '^https://openalex\.org/[WwAaSsIiCcPpFf]\d{3,13}$'), + url VARCHAR(500) CHECK (url ~ '^https?://\S+$'), + title VARCHAR(3000) NOT NULL, + authors VARCHAR(50000), + publisher VARCHAR(255), + publication_year SMALLINT, + journal VARCHAR(500), + page VARCHAR(50), + image_url VARCHAR(500) CHECK (image_url ~ '^https?://\S+$'), + mention_type mention_type NOT NULL, + source VARCHAR(50) NOT NULL, + version VARCHAR(100), + note VARCHAR(500), + scraped_at TIMESTAMPTZ, + citations_scraped_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE FUNCTION sanitise_insert_mention() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_mention BEFORE INSERT ON mention FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_mention(); + + +CREATE FUNCTION sanitise_update_mention() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_mention BEFORE UPDATE ON mention FOR EACH ROW EXECUTE PROCEDURE sanitise_update_mention(); + + +CREATE TABLE mention_for_software ( + mention UUID REFERENCES mention (id), + software UUID REFERENCES software (id), + PRIMARY KEY (mention, software) +); + +CREATE INDEX mention_for_software_software_idx ON mention_for_software(software); + + +CREATE TABLE reference_paper_for_software ( + mention UUID REFERENCES mention (id), + software UUID REFERENCES software (id), + PRIMARY KEY (mention, software) +); + +CREATE INDEX reference_paper_for_software_software_idx ON reference_paper_for_software(software); + + +CREATE TABLE citation_for_mention ( + mention UUID REFERENCES mention (id), + citation UUID REFERENCES mention (id), + PRIMARY KEY (mention, citation) +); + +CREATE INDEX citation_for_mention_citation_idx ON citation_for_mention(citation); + + +CREATE FUNCTION search_mentions_for_software(software_id UUID, search_text VARCHAR) RETURNS SETOF mention STABLE LANGUAGE plpgsql AS +$$ +BEGIN + RETURN QUERY SELECT * FROM mention + WHERE id NOT IN ( + SELECT mention FROM mention_for_software WHERE mention_for_software.software = software_id + ) + AND ( + url ILIKE CONCAT('%', search_text, '%') OR title ILIKE CONCAT('%', search_text, '%') + ); + RETURN; +END +$$; + + + +CREATE TABLE output_for_project ( + mention UUID REFERENCES mention (id), + project UUID REFERENCES project (id), + PRIMARY KEY (mention, project) +); + +CREATE INDEX output_for_project_project_idx ON output_for_project(project); + + +CREATE TABLE impact_for_project ( + mention UUID REFERENCES mention (id), + project UUID REFERENCES project (id), + PRIMARY KEY (mention, project) +); + +CREATE INDEX impact_for_project_project_idx ON impact_for_project(project); + + +CREATE FUNCTION search_impact_for_project(project_id UUID, search_text VARCHAR) RETURNS SETOF mention STABLE LANGUAGE plpgsql AS +$$ +BEGIN + RETURN QUERY SELECT * FROM mention + WHERE id NOT IN ( + SELECT mention FROM impact_for_project WHERE impact_for_project.project = project_id + ) + AND ( + url ILIKE CONCAT('%', search_text, '%') OR title ILIKE CONCAT('%', search_text, '%') + ); + RETURN; +END +$$; + + +CREATE FUNCTION search_output_for_project(project_id UUID, search_text VARCHAR) RETURNS SETOF mention STABLE LANGUAGE plpgsql AS +$$ +BEGIN + RETURN QUERY SELECT * FROM mention + WHERE id NOT IN ( + SELECT mention FROM output_for_project WHERE output_for_project.project = project_id + ) + AND ( + url ILIKE CONCAT('%', search_text, '%') OR title ILIKE CONCAT('%', search_text, '%') + ); + RETURN; +END +$$; diff --git a/database/012-create-release-table.sql b/database/012-create-release-table.sql new file mode 100644 index 0000000..a4bd3ad --- /dev/null +++ b/database/012-create-release-table.sql @@ -0,0 +1,55 @@ +-- SPDX-FileCopyrightText: 2021 - 2024 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2021 - 2024 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2022 dv4all +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE TABLE release ( + software UUID REFERENCES software (id) PRIMARY KEY, + releases_scraped_at TIMESTAMPTZ +); + + +CREATE TABLE release_version ( + release_id UUID REFERENCES release (software), + mention_id UUID REFERENCES mention (id), + PRIMARY KEY (release_id, mention_id) +); + +CREATE INDEX release_version_mention_id_idx ON release_version(mention_id); + + +CREATE FUNCTION z_delete_old_releases() RETURNS TRIGGER LANGUAGE plpgsql SECURITY DEFINER AS +$$ +BEGIN + IF NEW.concept_doi IS DISTINCT FROM OLD.concept_doi THEN + DELETE FROM release_version WHERE release_version.release_id = OLD.id; + DELETE FROM release WHERE release.software = OLD.id; + END IF; + RETURN NEW; +END +$$; + +CREATE TRIGGER z_delete_old_releases BEFORE UPDATE ON software FOR EACH ROW EXECUTE PROCEDURE z_delete_old_releases(); + + +CREATE FUNCTION software_join_release() RETURNS TABLE ( + software_id UUID, + slug VARCHAR, + concept_doi CITEXT, + versioned_dois CITEXT[], + releases_scraped_at TIMESTAMPTZ +) LANGUAGE plpgsql STABLE AS +$$ +BEGIN + RETURN QUERY + SELECT software.id AS software_id, software.slug, software.concept_doi, ARRAY_AGG(mention.doi), release.releases_scraped_at + FROM software + LEFT JOIN release ON software.id = release.software + LEFT JOIN release_version ON release_version.release_id = release.software + LEFT JOIN mention ON release_version.mention_id = mention.id + GROUP BY software.id, software.slug, software.concept_doi, release.software, release.releases_scraped_at; + RETURN; +END +$$; diff --git a/database/013-create-organisation-table.sql b/database/013-create-organisation-table.sql new file mode 100644 index 0000000..0e219ca --- /dev/null +++ b/database/013-create-organisation-table.sql @@ -0,0 +1,195 @@ +-- SPDX-FileCopyrightText: 2022 - 2023 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2022 - 2023 dv4all +-- SPDX-FileCopyrightText: 2022 - 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2022 - 2025 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) (dv4all) +-- SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +-- SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE FUNCTION varchar_array_to_string(arr VARCHAR[]) +RETURNS VARCHAR +LANGUAGE plpgsql +IMMUTABLE +AS +$$ + DECLARE entry VARCHAR; + DECLARE result VARCHAR := ''; + BEGIN + IF arr IS NULL THEN + RETURN NULL; + END IF; + FOREACH entry IN ARRAY arr LOOP + CONTINUE WHEN entry IS NULL; + IF result = '' THEN result := entry; ELSE result := result || ';' || entry; END IF; + END LOOP; + RETURN result; + END; +$$; + +CREATE TABLE organisation ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + parent UUID REFERENCES organisation (id), + primary_maintainer UUID REFERENCES account (id), + slug VARCHAR(200) NOT NULL CHECK (slug ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + name VARCHAR(200) NOT NULL, + short_description VARCHAR(300), + description VARCHAR(10000), + ror_id VARCHAR(100) UNIQUE CHECK (ror_id ~ '^https://ror\.org/(0[a-hj-km-np-tv-z|0-9]{6}[0-9]{2})$'), + website VARCHAR(200) UNIQUE, + is_tenant BOOLEAN DEFAULT FALSE NOT NULL, + country VARCHAR(100), + city VARCHAR(100), + wikipedia_url VARCHAR(300), + ror_names VARCHAR(200)[], + ror_names_string VARCHAR GENERATED ALWAYS AS (varchar_array_to_string(ror_names)) STORED, + ror_types VARCHAR(100)[], + lat float8, + lon float8, + ror_scraped_at TIMESTAMPTZ, + ror_last_error VARCHAR(500), + logo_id VARCHAR(40) REFERENCES image(id), + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + UNIQUE (slug, parent) +); + +CREATE INDEX organisation_parent_idx ON organisation(parent); +CREATE UNIQUE INDEX unique_slug_for_top_level_org_idx ON organisation(slug, (parent IS NULL)) WHERE parent IS NULL; +CREATE UNIQUE INDEX unique_name_and_parent_idx ON organisation(name, parent); + +CREATE FUNCTION check_cycle_organisations() RETURNS TRIGGER STABLE LANGUAGE plpgsql SECURITY DEFINER AS +$$ +DECLARE initial_org UUID = NEW.id; +DECLARE current_org UUID = NEW.parent; +BEGIN + WHILE current_org IS NOT NULL LOOP + IF current_org = initial_org THEN + RAISE EXCEPTION USING MESSAGE = 'Cycle detected for organisation with id ' || NEW.id; + END IF; + SELECT parent FROM organisation WHERE id = current_org INTO current_org; + END LOOP; + RETURN NEW; +END +$$; + +-- z_ prefix so that if is executed after the sanitise_update_organisation trigger +CREATE TRIGGER z_check_cycle_organisations BEFORE UPDATE OF parent ON organisation FOR EACH ROW EXECUTE PROCEDURE check_cycle_organisations(); + +CREATE FUNCTION sanitise_insert_organisation() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + + IF CURRENT_USER = 'rsd_admin' OR (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) THEN + RETURN NEW; + END IF; + + IF NOT NEW.is_tenant AND NEW.parent IS NULL AND NEW.primary_maintainer IS NULL THEN + RETURN NEW; + END IF; + + IF (SELECT primary_maintainer FROM organisation o WHERE o.id = NEW.parent) = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account') + AND + NEW.primary_maintainer = (SELECT primary_maintainer FROM organisation o WHERE o.id = NEW.parent) + THEN + RETURN NEW; + END IF; + + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to add this organisation'; +END +$$; + +CREATE TRIGGER sanitise_insert_organisation BEFORE INSERT ON organisation FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_organisation(); + + +CREATE FUNCTION sanitise_update_organisation() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + + IF NEW.slug IS DISTINCT FROM OLD.slug AND CURRENT_USER IS DISTINCT FROM 'rsd_admin' AND (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) IS DISTINCT FROM TRUE THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to change the slug'; + END IF; + + IF CURRENT_USER <> 'rsd_admin' AND NOT (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) THEN + IF NEW.is_tenant IS DISTINCT FROM OLD.is_tenant OR NEW.primary_maintainer IS DISTINCT FROM OLD.primary_maintainer THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to change the tenant status or primary maintainer for organisation ' || OLD.name; + END IF; + END IF; + + RETURN NEW; +END +$$; + +CREATE TRIGGER sanitise_update_organisation BEFORE UPDATE ON organisation FOR EACH ROW EXECUTE PROCEDURE sanitise_update_organisation(); + + +-- including the parent itself +CREATE FUNCTION list_child_organisations(parent_id UUID) RETURNS TABLE (organisation_id UUID, organisation_name VARCHAR) STABLE LANGUAGE sql AS +$$ +WITH RECURSIVE search_tree(id, name) AS ( + SELECT o.id, o.name + FROM organisation o WHERE id = parent_id + UNION ALL + SELECT o.id, o.name + FROM organisation o, search_tree st + WHERE o.parent = st.id +) +SELECT * FROM search_tree; +$$; + + +CREATE FUNCTION list_parent_organisations(id UUID) RETURNS TABLE (slug VARCHAR, organisation_id UUID) STABLE LANGUAGE sql AS +$$ +WITH RECURSIVE search_tree(slug, organisation_id, parent) AS ( + SELECT o.slug, o.id, o.parent + FROM organisation o WHERE o.id = list_parent_organisations.id + UNION ALL + SELECT o.slug, o.id, o.parent + FROM organisation o, search_tree st + WHERE o.id = st.parent +) +SELECT slug, organisation_id FROM search_tree; +$$; + + +CREATE FUNCTION slug_to_organisation(full_slug VARCHAR) RETURNS UUID STABLE LANGUAGE plpgsql AS +$$ +DECLARE current_org UUID; +DECLARE slug_part VARCHAR; +BEGIN + FOREACH slug_part IN ARRAY string_to_array(full_slug, '/') LOOP + SELECT id FROM organisation WHERE (parent = current_org OR (parent IS NULL AND current_org IS NULL)) AND slug = slug_part INTO current_org; + IF (current_org IS NULL) THEN + RETURN NULL; + END IF; + END LOOP; + RETURN current_org; +END +$$; + + +-- ORGANISATION route / path for all organisations +-- we combine slugs of all parent organisation into route +CREATE FUNCTION organisation_route(id UUID) RETURNS TABLE ( + organisation UUID, + rsd_path VARCHAR, + parent_names VARCHAR +) STABLE LANGUAGE sql AS +$$ +WITH RECURSIVE search_tree(slug, name, organisation_id, parent, reverse_depth) AS ( + SELECT o.slug, o.name, o.id, o.parent, 1 + FROM organisation o WHERE o.id = organisation_route.id + UNION ALL + SELECT o.slug, o.name, o.id, o.parent, st.reverse_depth + 1 + FROM organisation o, search_tree st + WHERE o.id = st.parent +) +SELECT organisation_route.id, STRING_AGG(slug, '/' ORDER BY reverse_depth DESC), STRING_AGG(name, ' > ' ORDER BY reverse_depth DESC) FROM search_tree; +$$; diff --git a/database/014-create-keyword-and-category.sql b/database/014-create-keyword-and-category.sql new file mode 100644 index 0000000..f0073e3 --- /dev/null +++ b/database/014-create-keyword-and-category.sql @@ -0,0 +1,275 @@ +-- SPDX-FileCopyrightText: 2022 - 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2022 - 2025 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2023 - 2024 Felix Mühlbauer (GFZ) +-- SPDX-FileCopyrightText: 2023 - 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +-- SPDX-FileCopyrightText: 2024 - 2025 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +-- +-- SPDX-License-Identifier: Apache-2.0 + +-------------- +-- Keywords -- +-------------- + +CREATE TABLE keyword ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + value CITEXT UNIQUE CHECK (value ~ '^\S+( \S+)*$') +); + +CREATE FUNCTION sanitise_insert_keyword() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_keyword BEFORE INSERT ON keyword FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_keyword(); + + +CREATE FUNCTION sanitise_update_keyword() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_keyword BEFORE UPDATE ON keyword FOR EACH ROW EXECUTE PROCEDURE sanitise_update_keyword(); + + +CREATE TABLE keyword_for_software ( + software UUID REFERENCES software (id), + keyword UUID REFERENCES keyword (id), + PRIMARY KEY (software, keyword) +); + +CREATE INDEX keyword_for_software_keyword_idx ON keyword_for_software(keyword); + +CREATE TABLE keyword_for_project ( + project UUID REFERENCES project (id), + keyword UUID REFERENCES keyword (id), + PRIMARY KEY (project, keyword) +); + +CREATE INDEX keyword_for_project_keyword_idx ON keyword_for_project(keyword); + +-- ADD basic keywords from topics and tags +INSERT into keyword (value) +VALUES + ('Big data'), + ('GPU'), + ('High performance computing'), + ('Image processing'), + ('Inter-operability & linked data'), + ('Machine learning'), + ('Multi-scale & multi model simulations'), + ('Optimized data handling'), + ('Real time data analysis'), + ('Text analysis & natural language processing'), + ('Visualization'), + ('Workflow technologies'); + + +-- KEYWORDS for community +CREATE TABLE keyword_for_community ( + community UUID REFERENCES community (id), + keyword UUID REFERENCES keyword (id), + PRIMARY KEY (community, keyword) +); + +CREATE INDEX keyword_for_community_keyword_idx ON keyword_for_community(keyword); + + +---------------- +-- Categories -- +---------------- + +CREATE TABLE category ( + id UUID PRIMARY KEY, + parent UUID REFERENCES category DEFAULT NULL, + community UUID REFERENCES community(id) DEFAULT NULL, + organisation UUID REFERENCES organisation(id) DEFAULT NULL, + allow_software BOOLEAN NOT NULL DEFAULT FALSE, + allow_projects BOOLEAN NOT NULL DEFAULT FALSE, + short_name VARCHAR(100) NOT NULL, + name VARCHAR(250) NOT NULL, + properties JSONB NOT NULL DEFAULT '{}'::jsonb, + provenance_iri VARCHAR(250) DEFAULT NULL, -- e.g. https://www.w3.org/TR/skos-reference/#mapping + + CONSTRAINT only_one_entity CHECK (community IS NULL OR organisation IS NULL), + CONSTRAINT unique_short_name UNIQUE NULLS NOT DISTINCT (parent, short_name, community, organisation), + CONSTRAINT unique_name UNIQUE NULLS NOT DISTINCT (parent, name, community, organisation), + CONSTRAINT invalid_value_for_properties CHECK (properties - '{icon, is_highlight, description, subtitle, tree_level_labels}'::text[] = '{}'::jsonb), + CONSTRAINT highlight_must_be_top_level_category CHECK (NOT ((properties->>'is_highlight')::boolean AND parent IS NOT NULL)) +); + +CREATE INDEX category_parent_idx ON category(parent); +CREATE INDEX category_community_idx ON category(community); +CREATE INDEX category_organisation_idx ON category(organisation); + +-- sanitize categories + +CREATE FUNCTION sanitise_insert_category() +RETURNS TRIGGER +LANGUAGE plpgsql +SECURITY DEFINER AS +$$ +BEGIN + IF NEW.id IS NOT NULL THEN + RAISE EXCEPTION USING MESSAGE = 'The category id is generated automatically and may not be set.'; + END IF; + IF NEW.parent IS NOT NULL AND (SELECT community FROM category WHERE id = NEW.parent) IS DISTINCT FROM NEW.community THEN + RAISE EXCEPTION USING MESSAGE = 'The community must be the same as of its parent.'; + END IF; + IF NEW.parent IS NOT NULL AND (SELECT organisation FROM category WHERE id = NEW.parent) IS DISTINCT FROM NEW.organisation THEN + RAISE EXCEPTION USING MESSAGE = 'The organisation must be the same as of its parent.'; + END IF; + NEW.id = gen_random_uuid(); + RETURN NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_category + BEFORE INSERT ON category + FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_category(); + + +CREATE FUNCTION sanitise_update_category() +RETURNS TRIGGER +LANGUAGE plpgsql +SECURITY DEFINER AS +$$ +BEGIN + IF NEW.id != OLD.id THEN + RAISE EXCEPTION USING MESSAGE = 'The category id may not be changed.'; + END IF; + IF NEW.community IS DISTINCT FROM OLD.community THEN + RAISE EXCEPTION USING MESSAGE = 'The community this category belongs to may not be changed.'; + END IF; + IF NEW.parent IS NOT NULL AND (SELECT community FROM category WHERE id = NEW.parent) IS DISTINCT FROM NEW.community THEN + RAISE EXCEPTION USING MESSAGE = 'The community must be the same as of its parent.'; + END IF; + IF NEW.organisation IS DISTINCT FROM OLD.organisation THEN + RAISE EXCEPTION USING MESSAGE = 'The organisation this category belongs to may not be changed.'; + END IF; + IF NEW.parent IS NOT NULL AND (SELECT organisation FROM category WHERE id = NEW.parent) IS DISTINCT FROM NEW.organisation THEN + RAISE EXCEPTION USING MESSAGE = 'The organisation must be the same as of its parent.'; + END IF; + RETURN NEW; +END +$$; + +CREATE TRIGGER sanitise_update_category + BEFORE UPDATE ON category + FOR EACH ROW EXECUTE PROCEDURE sanitise_update_category(); + + +CREATE FUNCTION check_cycle_categories() +RETURNS TRIGGER STABLE +LANGUAGE plpgsql +SECURITY DEFINER AS +$$ + DECLARE first_id UUID = NEW.id; + DECLARE current_id UUID = NEW.parent; +BEGIN + WHILE current_id IS NOT NULL LOOP + IF current_id = first_id THEN + RAISE EXCEPTION USING MESSAGE = 'Cycle detected for category with id ' || NEW.id; + END IF; + SELECT parent FROM category WHERE id = current_id INTO current_id; + END LOOP; + RETURN NEW; +END +$$; + +CREATE TRIGGER zzz_check_cycle_categories -- triggers are executed in alphabetical order + AFTER INSERT OR UPDATE ON category + FOR EACH ROW EXECUTE PROCEDURE check_cycle_categories(); + + +-- helper functions + +CREATE FUNCTION category_path(category_id UUID) +RETURNS TABLE ( + id UUID, + parent UUID, + community UUID, + organisation UUID, + short_name VARCHAR, + name VARCHAR, + properties JSONB, + provenance_iri VARCHAR +) +LANGUAGE SQL STABLE AS +$$ + WITH RECURSIVE cat_path AS ( + SELECT *, 1 AS r_index + FROM category WHERE id = category_id + UNION ALL + SELECT category.*, cat_path.r_index+1 + FROM category + JOIN cat_path + ON category.id = cat_path.parent + ) + -- 1. How can we reverse the output rows without injecting a new column (r_index)? + -- 2. How a table row "type" could be used here Now we have to list all columns of `category` explicitly + -- I want to have something like `* without 'r_index'` to be independent from modifications of `category` + -- 3. Maybe this could be improved by using SEARCH keyword. + SELECT id, parent, community, organisation, short_name, name, properties, provenance_iri + FROM cat_path + ORDER BY r_index DESC; +$$; + + +-- TABLE FOR software categories +-- includes organisation, community and general categories +-- Note! to filter specific categories of an community or organisation use join with community table + +-- if you ever change this table, then software_of_category() needs to be changed as well +CREATE TABLE category_for_software ( + software_id UUID REFERENCES software (id), + category_id UUID REFERENCES category (id), + PRIMARY KEY (software_id, category_id) +); + +CREATE INDEX category_for_software_category_id_idx ON category_for_software(category_id); + + +-- TABLE FOR project categories +-- currently used only for organisation categories + +-- if you ever change this table, then projects_of_category() needs to be changed as well +CREATE TABLE category_for_project ( + project_id UUID REFERENCES project (id), + category_id UUID REFERENCES category (id), + PRIMARY KEY (project_id, category_id) +); + +CREATE INDEX category_for_project_category_id_idx ON category_for_project(category_id); + + +CREATE FUNCTION software_of_category() +RETURNS SETOF category_for_software +LANGUAGE SQL STABLE AS +$$ +SELECT DISTINCT software.id, category_path.id +FROM software +INNER JOIN category_for_software + ON software.id = category_for_software.software_id +INNER JOIN category_path(category_for_software.category_id) + ON TRUE; +$$; + + +CREATE FUNCTION projects_of_category() +RETURNS SETOF category_for_project +LANGUAGE SQL STABLE AS +$$ +SELECT DISTINCT project.id, category_path.id +FROM project +INNER JOIN category_for_project + ON project.id = category_for_project.project_id +INNER JOIN category_path(category_for_project.category_id) + ON TRUE; +$$; diff --git a/database/015-inter-relation-tables.sql b/database/015-inter-relation-tables.sql new file mode 100644 index 0000000..afae71a --- /dev/null +++ b/database/015-inter-relation-tables.sql @@ -0,0 +1,174 @@ +-- SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2022 dv4all +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE TYPE relation_status AS ENUM ( + 'rejected_by_origin', + 'rejected_by_relation', + 'approved' +); + +CREATE TYPE organisation_role AS ENUM ( + 'funding', + 'hosting', + 'participating' +); + +CREATE TABLE software_for_software ( + origin UUID REFERENCES software (id), + relation UUID REFERENCES software (id), + PRIMARY KEY (origin, relation) +); + +CREATE FUNCTION sanitise_update_software_for_software() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.origin = OLD.origin; + NEW.relation = OLD.relation; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_software_for_software BEFORE UPDATE ON software_for_software FOR EACH ROW EXECUTE PROCEDURE sanitise_update_software_for_software(); + + +CREATE TABLE software_for_project ( + software UUID REFERENCES software (id), + project UUID REFERENCES project (id), + status relation_status NOT NULL DEFAULT 'approved', + PRIMARY KEY (software, project) +); + +CREATE FUNCTION sanitise_update_software_for_project() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.software = OLD.software; + NEW.project = OLD.project; + + IF CURRENT_USER <> 'rsd_admin' AND NOT (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) THEN + IF NEW.status <> OLD.status AND ( + (OLD.status = 'rejected_by_origin' AND (NEW.status <> 'approved' OR OLD.software NOT IN (SELECT * FROM software_of_current_maintainer()))) OR + (OLD.status = 'rejected_by_relation' AND (NEW.status <> 'approved' OR OLD.project NOT IN (SELECT * FROM projects_of_current_maintainer()))) OR + (OLD.status = 'approved' AND NEW.status = 'rejected_by_origin' AND OLD.software NOT IN (SELECT * FROM projects_of_current_maintainer())) OR + (OLD.status = 'approved' AND NEW.status = 'rejected_by_relation' AND OLD.project NOT IN (SELECT * FROM software_of_current_maintainer())) + ) THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to make this change to the status value for software ' || OLD.software || ' and project ' || OLD.project; + END IF; + END IF; + + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_software_for_project BEFORE UPDATE ON software_for_project FOR EACH ROW EXECUTE PROCEDURE sanitise_update_software_for_project(); + + +CREATE TABLE project_for_project ( + origin UUID REFERENCES project (id), + relation UUID REFERENCES project (id), + status relation_status NOT NULL DEFAULT 'approved', + PRIMARY KEY (origin, relation) +); + +CREATE FUNCTION sanitise_update_project_for_project() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.origin = OLD.origin; + NEW.relation = OLD.relation; + + IF CURRENT_USER <> 'rsd_admin' AND NOT (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) THEN + IF NEW.status <> OLD.status AND ( + (OLD.status = 'rejected_by_origin' AND (NEW.status <> 'approved' OR OLD.origin NOT IN (SELECT * FROM projects_of_current_maintainer()))) OR + (OLD.status = 'rejected_by_relation' AND (NEW.status <> 'approved' OR OLD.relation NOT IN (SELECT * FROM projects_of_current_maintainer()))) OR + (OLD.status = 'approved' AND NEW.status = 'rejected_by_origin' AND OLD.origin NOT IN (SELECT * FROM projects_of_current_maintainer())) OR + (OLD.status = 'approved' AND NEW.status = 'rejected_by_relation' AND OLD.relation NOT IN (SELECT * FROM projects_of_current_maintainer())) + ) THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to make this change to the status value for origin ' || OLD.origin || ' and relation ' || OLD.relation; + END IF; + END IF; + + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_project_for_project BEFORE UPDATE ON project_for_project FOR EACH ROW EXECUTE PROCEDURE sanitise_update_project_for_project(); + + +CREATE TABLE software_for_organisation ( + software UUID REFERENCES software (id), + organisation UUID REFERENCES organisation (id), + status relation_status NOT NULL DEFAULT 'approved', + is_featured BOOLEAN DEFAULT FALSE NOT NULL, + position INTEGER, + PRIMARY KEY (software, organisation) +); + +CREATE INDEX software_for_organisation_organisation_idx ON software_for_organisation(organisation); + +CREATE FUNCTION sanitise_update_software_for_organisation() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.software = OLD.software; + NEW.organisation = OLD.organisation; + + IF CURRENT_USER <> 'rsd_admin' AND NOT (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) THEN + IF NEW.is_featured <> OLD.is_featured AND OLD.organisation NOT IN (SELECT * FROM organisations_of_current_maintainer()) THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to change the is_featured value for software ' || OLD.software || 'and organisation ' || OLD.organisation; + END IF; + IF NEW.status <> OLD.status AND ( + (OLD.status = 'rejected_by_origin' AND (NEW.status <> 'approved' OR OLD.software NOT IN (SELECT * FROM software_of_current_maintainer()))) OR + (OLD.status = 'rejected_by_relation' AND (NEW.status <> 'approved' OR OLD.organisation NOT IN (SELECT * FROM organisations_of_current_maintainer()))) OR + (OLD.status = 'approved' AND NEW.status = 'rejected_by_origin' AND OLD.software NOT IN (SELECT * FROM software_of_current_maintainer())) OR + (OLD.status = 'approved' AND NEW.status = 'rejected_by_relation' AND OLD.organisation NOT IN (SELECT * FROM organisations_of_current_maintainer())) + ) THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to make this change to the status value for software ' || OLD.software || ' and organisation ' || OLD.organisation; + END IF; + END IF; + + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_software_for_organisation BEFORE UPDATE ON software_for_organisation FOR EACH ROW EXECUTE PROCEDURE sanitise_update_software_for_organisation(); + + +CREATE TABLE project_for_organisation ( + project UUID REFERENCES project (id), + organisation UUID REFERENCES organisation (id), + status relation_status NOT NULL DEFAULT 'approved', + role organisation_role NOT NULL DEFAULT 'participating', + is_featured BOOLEAN DEFAULT FALSE NOT NULL, + position INTEGER, + PRIMARY KEY (project, organisation, role) +); + +CREATE INDEX project_for_organisation_organisation_idx ON project_for_organisation(organisation); + +CREATE FUNCTION sanitise_update_project_for_organisation() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.project = OLD.project; + NEW.organisation = OLD.organisation; + + IF CURRENT_USER <> 'rsd_admin' AND NOT (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) THEN + IF NEW.is_featured <> OLD.is_featured AND OLD.organisation NOT IN (SELECT * FROM organisations_of_current_maintainer()) THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to change the is_featured value for project ' || OLD.project || 'and organisation ' || OLD.organisation; + END IF; + IF NEW.status <> OLD.status AND ( + (OLD.status = 'rejected_by_origin' AND (NEW.status <> 'approved' OR OLD.project NOT IN (SELECT * FROM projects_of_current_maintainer()))) OR + (OLD.status = 'rejected_by_relation' AND (NEW.status <> 'approved' OR OLD.organisation NOT IN (SELECT * FROM organisations_of_current_maintainer()))) OR + (OLD.status = 'approved' AND NEW.status = 'rejected_by_origin' AND OLD.project NOT IN (SELECT * FROM projects_of_current_maintainer())) OR + (OLD.status = 'approved' AND NEW.status = 'rejected_by_relation' AND OLD.organisation NOT IN (SELECT * FROM organisations_of_current_maintainer())) + ) THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to make this change to the status value for project ' || OLD.project || ' and organisation ' || OLD.organisation; + END IF; + END IF; + + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_project_for_organisation BEFORE UPDATE ON project_for_organisation FOR EACH ROW EXECUTE PROCEDURE sanitise_update_project_for_organisation(); diff --git a/database/016-create-maintainer-tables.sql b/database/016-create-maintainer-tables.sql new file mode 100644 index 0000000..ffc428d --- /dev/null +++ b/database/016-create-maintainer-tables.sql @@ -0,0 +1,277 @@ +-- SPDX-FileCopyrightText: 2021 - 2024 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2021 - 2024 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2022 dv4all +-- SPDX-FileCopyrightText: 2024 Dusan Mijatovic (Netherlands eScience Center) +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE TABLE maintainer_for_software ( + maintainer UUID REFERENCES account (id), + software UUID REFERENCES software (id), + PRIMARY KEY (maintainer, software) +); + +CREATE TABLE maintainer_for_project ( + maintainer UUID REFERENCES account (id), + project UUID REFERENCES project (id), + PRIMARY KEY (maintainer, project) +); + +CREATE TABLE maintainer_for_organisation ( + maintainer UUID REFERENCES account (id), + organisation UUID REFERENCES organisation (id), + PRIMARY KEY (maintainer, organisation) +); + +-- PROJECT +-- create table for maintainer's "magic link" invitations +CREATE TABLE invite_maintainer_for_project ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + project UUID REFERENCES project (id) NOT NULL, + created_by UUID REFERENCES account (id), + claimed_by UUID REFERENCES account (id), + claimed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT LOCALTIMESTAMP, + expires_at TIMESTAMPTZ NOT NULL GENERATED ALWAYS AS (created_at AT TIME ZONE 'UTC' + INTERVAL '31 days') STORED +); + +CREATE FUNCTION sanitise_insert_invite_maintainer_for_project() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.claimed_by = NULL; + NEW.claimed_at = NULL; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_invite_maintainer_for_project BEFORE INSERT ON invite_maintainer_for_project FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_invite_maintainer_for_project(); + + +CREATE FUNCTION sanitise_update_invite_maintainer_for_project() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.project = OLD.project; + NEW.created_by = OLD.created_by; + NEW.created_at = OLD.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_invite_maintainer_for_project BEFORE UPDATE ON invite_maintainer_for_project FOR EACH ROW EXECUTE PROCEDURE sanitise_update_invite_maintainer_for_project(); + + +CREATE FUNCTION accept_invitation_project(invitation UUID) RETURNS TABLE(title VARCHAR, slug VARCHAR) LANGUAGE plpgsql VOLATILE SECURITY DEFINER AS +$$ +DECLARE invitation_row invite_maintainer_for_project%ROWTYPE; +DECLARE account UUID; +BEGIN + account = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'); + IF account IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please login first'; + END IF; + + IF invitation IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please provide an invitation id'; + END IF; + + SELECT * FROM invite_maintainer_for_project WHERE id = invitation INTO invitation_row; + IF invitation_row.id IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Invitation with id ' || invitation || ' does not exist'; + END IF; + + IF invitation_row.claimed_by IS NOT NULL OR invitation_row.claimed_at IS NOT NULL OR invitation_row.expires_at < CURRENT_TIMESTAMP THEN + RAISE EXCEPTION USING MESSAGE = 'Invitation with id ' || invitation || ' is expired'; + END IF; + +-- Only use the invitation if not already a maintainer + IF NOT EXISTS(SELECT 1 FROM maintainer_for_project WHERE maintainer = account AND project = invitation_row.project) THEN + UPDATE invite_maintainer_for_project SET claimed_by = account, claimed_at = LOCALTIMESTAMP WHERE id = invitation; + INSERT INTO maintainer_for_project VALUES (account, invitation_row.project); + END IF; + + RETURN QUERY + SELECT project.title, project.slug FROM project WHERE project.id = invitation_row.project; + RETURN; +END +$$; + +-- SOFTWARE +-- create tables and rpc for maintainer's "magic link" invitations +CREATE TABLE invite_maintainer_for_software ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + software UUID REFERENCES software (id) NOT NULL, + created_by UUID REFERENCES account (id), + claimed_by UUID REFERENCES account (id), + claimed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT LOCALTIMESTAMP, + expires_at TIMESTAMPTZ NOT NULL GENERATED ALWAYS AS (created_at AT TIME ZONE 'UTC' + INTERVAL '31 days') STORED +); + +CREATE FUNCTION sanitise_insert_invite_maintainer_for_software() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.claimed_by = NULL; + NEW.claimed_at = NULL; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_invite_maintainer_for_software BEFORE INSERT ON invite_maintainer_for_software FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_invite_maintainer_for_software(); + + +CREATE FUNCTION sanitise_update_invite_maintainer_for_software() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.software = OLD.software; + NEW.created_by = OLD.created_by; + NEW.created_at = OLD.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_invite_maintainer_for_software BEFORE UPDATE ON invite_maintainer_for_software FOR EACH ROW EXECUTE PROCEDURE sanitise_update_invite_maintainer_for_software(); + + +CREATE FUNCTION accept_invitation_software(invitation UUID) RETURNS TABLE( + brand_name VARCHAR, + slug VARCHAR +) LANGUAGE plpgsql VOLATILE SECURITY DEFINER AS +$$ +DECLARE invitation_row invite_maintainer_for_software%ROWTYPE; +DECLARE account UUID; +BEGIN + account = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'); + IF account IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please login first'; + END IF; + + IF invitation IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please provide an invitation id'; + END IF; + + SELECT * FROM invite_maintainer_for_software WHERE id = invitation INTO invitation_row; + IF invitation_row.id IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Invitation with id ' || invitation || ' does not exist'; + END IF; + + IF invitation_row.claimed_by IS NOT NULL OR invitation_row.claimed_at IS NOT NULL OR invitation_row.expires_at < CURRENT_TIMESTAMP THEN + RAISE EXCEPTION USING MESSAGE = 'Invitation with id ' || invitation || ' is expired'; + END IF; + +-- Only use the invitation if not already a maintainer + IF NOT EXISTS(SELECT 1 FROM maintainer_for_software WHERE maintainer = account AND software = invitation_row.software) THEN + UPDATE invite_maintainer_for_software SET claimed_by = account, claimed_at = LOCALTIMESTAMP WHERE id = invitation; + INSERT INTO maintainer_for_software VALUES (account, invitation_row.software); + END IF; + + RETURN QUERY + SELECT software.brand_name, software.slug FROM software WHERE software.id = invitation_row.software; + RETURN; +END +$$; + +-- ORGANISATION +-- create tables and rpc for maintainer's "magic link" invitations +CREATE TABLE invite_maintainer_for_organisation ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + organisation UUID REFERENCES organisation (id) NOT NULL, + created_by UUID REFERENCES account (id), + claimed_by UUID REFERENCES account (id), + claimed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT LOCALTIMESTAMP, + expires_at TIMESTAMPTZ NOT NULL GENERATED ALWAYS AS (created_at AT TIME ZONE 'UTC' + INTERVAL '31 days') STORED +); + +CREATE FUNCTION sanitise_insert_invite_maintainer_for_organisation() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.claimed_by = NULL; + NEW.claimed_at = NULL; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_invite_maintainer_for_organisation BEFORE INSERT ON invite_maintainer_for_organisation FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_invite_maintainer_for_organisation(); + + +CREATE FUNCTION sanitise_update_invite_maintainer_for_organisation() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.organisation = OLD.organisation; + NEW.created_by = OLD.created_by; + NEW.created_at = OLD.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_invite_maintainer_for_organisation BEFORE UPDATE ON invite_maintainer_for_organisation FOR EACH ROW EXECUTE PROCEDURE sanitise_update_invite_maintainer_for_organisation(); + + +CREATE FUNCTION accept_invitation_organisation(invitation UUID) RETURNS TABLE( + id UUID, + name VARCHAR +) LANGUAGE plpgsql VOLATILE SECURITY DEFINER AS +$$ +DECLARE invitation_row invite_maintainer_for_organisation%ROWTYPE; +DECLARE account UUID; +BEGIN + account = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'); + IF account IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please login first'; + END IF; + + IF invitation IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please provide an invitation id'; + END IF; + + SELECT * FROM invite_maintainer_for_organisation WHERE invite_maintainer_for_organisation.id = invitation INTO invitation_row; + IF invitation_row.id IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Invitation with id ' || invitation || ' does not exist'; + END IF; + + IF invitation_row.claimed_by IS NOT NULL OR invitation_row.claimed_at IS NOT NULL OR invitation_row.expires_at < CURRENT_TIMESTAMP THEN + RAISE EXCEPTION USING MESSAGE = 'Invitation with id ' || invitation || ' is expired'; + END IF; + +-- Only use the invitation if not already a maintainer + IF NOT EXISTS( + SELECT + maintainer_for_organisation.maintainer + FROM + maintainer_for_organisation + WHERE + maintainer_for_organisation.maintainer=account AND maintainer_for_organisation.organisation=invitation_row.organisation + UNION + SELECT + organisation.primary_maintainer AS maintainer + FROM + organisation + WHERE + organisation.primary_maintainer=account AND organisation.id=invitation_row.organisation + LIMIT 1 + ) THEN + UPDATE invite_maintainer_for_organisation SET claimed_by = account, claimed_at = LOCALTIMESTAMP WHERE invite_maintainer_for_organisation.id = invitation; + INSERT INTO maintainer_for_organisation VALUES (account, invitation_row.organisation); + END IF; + + RETURN QUERY + SELECT + organisation.id, + organisation.name + FROM + organisation + WHERE + organisation.id = invitation_row.organisation; + RETURN; +END +$$; diff --git a/database/017-meta-pages.sql b/database/017-meta-pages.sql new file mode 100644 index 0000000..ad683b7 --- /dev/null +++ b/database/017-meta-pages.sql @@ -0,0 +1,56 @@ +-- SPDX-FileCopyrightText: 2021 - 2024 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2021 - 2024 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2022 dv4all +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- CUSTOM META PAGES TO BE ADDED TO RSD +-- THE PAGES ARE LISTED IN THE FOOTER + +CREATE TABLE meta_pages ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + slug VARCHAR(100) UNIQUE NOT NULL CHECK (slug ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + title VARCHAR(100) NOT NULL, + description VARCHAR(30000), + is_published BOOLEAN DEFAULT FALSE NOT NULL, + position INTEGER, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + + +CREATE FUNCTION sanitise_insert_meta_pages() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_meta_pages BEFORE INSERT ON meta_pages + FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_meta_pages(); + + +CREATE FUNCTION sanitise_update_meta_pages() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + + IF NEW.slug IS DISTINCT FROM OLD.slug AND CURRENT_USER IS DISTINCT FROM 'rsd_admin' AND ( + SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER + ) IS DISTINCT FROM TRUE + THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to change the slug'; + END IF; + + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_meta_pages BEFORE UPDATE ON meta_pages + FOR EACH ROW EXECUTE PROCEDURE sanitise_update_meta_pages(); diff --git a/database/018-create-backend-logs-table.sql b/database/018-create-backend-logs-table.sql new file mode 100644 index 0000000..dc711b2 --- /dev/null +++ b/database/018-create-backend-logs-table.sql @@ -0,0 +1,40 @@ +-- SPDX-FileCopyrightText: 2023 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2023 Netherlands eScience Center +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE TABLE backend_log ( + id UUID PRIMARY KEY, + service_name VARCHAR, + table_name VARCHAR, + reference_id UUID, + message VARCHAR, + stack_trace VARCHAR, + other_data JSONB, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE FUNCTION sanitise_insert_backend_log() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_backend_log BEFORE INSERT ON backend_log FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_backend_log(); + +CREATE FUNCTION sanitise_update_backend_log() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_backend_log BEFORE UPDATE ON backend_log FOR EACH ROW EXECUTE PROCEDURE sanitise_update_backend_log(); diff --git a/database/020-row-level-security.sql b/database/020-row-level-security.sql new file mode 100644 index 0000000..d2a5636 --- /dev/null +++ b/database/020-row-level-security.sql @@ -0,0 +1,816 @@ +-- SPDX-FileCopyrightText: 2021 - 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2021 - 2025 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2022 - 2023 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2022 - 2023 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +-- SPDX-FileCopyrightText: 2022 - 2023 dv4all +-- SPDX-FileCopyrightText: 2022 Christian Meeßen (GFZ) +-- SPDX-FileCopyrightText: 2023 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2023 Felix Mühlbauer (GFZ) +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- maintainer tables +ALTER TABLE maintainer_for_software ENABLE ROW LEVEL SECURITY; + +CREATE FUNCTION software_of_current_maintainer() RETURNS SETOF UUID STABLE LANGUAGE plpgsql SECURITY DEFINER AS +$$ +BEGIN + RETURN QUERY SELECT software FROM maintainer_for_software WHERE maintainer = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'); + RETURN; +END +$$; + +CREATE FUNCTION related_software() RETURNS SETOF UUID STABLE LANGUAGE plpgsql SECURITY DEFINER AS +$$ +BEGIN + RETURN QUERY SELECT software FROM software_for_organisation WHERE organisation IN (SELECT * FROM organisations_of_current_maintainer()); + RETURN QUERY SELECT software FROM software_for_project WHERE project IN (SELECT * FROM projects_of_current_maintainer()); + RETURN; +END +$$; + +CREATE POLICY maintainer_select ON maintainer_for_software FOR SELECT TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY maintainer_delete ON maintainer_for_software FOR DELETE TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY maintainer_insert ON maintainer_for_software FOR INSERT TO rsd_user + WITH CHECK (software IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON maintainer_for_software TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE maintainer_for_project ENABLE ROW LEVEL SECURITY; + +CREATE FUNCTION projects_of_current_maintainer() RETURNS SETOF UUID STABLE LANGUAGE plpgsql SECURITY DEFINER AS +$$ +BEGIN + RETURN QUERY SELECT project FROM maintainer_for_project WHERE maintainer = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'); + RETURN; +END +$$; + +CREATE FUNCTION related_projects() RETURNS SETOF UUID STABLE LANGUAGE plpgsql SECURITY DEFINER AS +$$ +BEGIN + RETURN QUERY SELECT project FROM project_for_organisation WHERE organisation IN (SELECT * FROM organisations_of_current_maintainer()); + RETURN QUERY SELECT project FROM software_for_project WHERE software IN (SELECT * FROM software_of_current_maintainer()); + RETURN; +END +$$; + +CREATE POLICY maintainer_select ON maintainer_for_project FOR SELECT TO rsd_user + USING (project IN (SELECT * FROM projects_of_current_maintainer())); + +CREATE POLICY maintainer_delete ON maintainer_for_project FOR DELETE TO rsd_user + USING (project IN (SELECT * FROM projects_of_current_maintainer())); + +CREATE POLICY maintainer_insert ON maintainer_for_project FOR INSERT TO rsd_user + WITH CHECK (project IN (SELECT * FROM projects_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON maintainer_for_project TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE maintainer_for_organisation ENABLE ROW LEVEL SECURITY; + +CREATE FUNCTION organisations_of_current_maintainer() RETURNS SETOF UUID STABLE LANGUAGE plpgsql SECURITY DEFINER AS +$$ +BEGIN + RETURN QUERY SELECT id FROM organisation WHERE primary_maintainer = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'); + RETURN QUERY SELECT organisation FROM maintainer_for_organisation WHERE maintainer = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'); + RETURN; +END +$$; + +CREATE POLICY maintainer_select ON maintainer_for_organisation FOR SELECT TO rsd_user + USING (organisation IN (SELECT * FROM organisations_of_current_maintainer())); + +CREATE POLICY maintainer_delete ON maintainer_for_organisation FOR DELETE TO rsd_user + USING (organisation IN (SELECT * FROM organisations_of_current_maintainer())); + +CREATE POLICY maintainer_insert ON maintainer_for_organisation FOR INSERT TO rsd_user + WITH CHECK (organisation IN (SELECT * FROM organisations_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON maintainer_for_organisation TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- invitations PROJECT +ALTER TABLE invite_maintainer_for_project ENABLE ROW LEVEL SECURITY; + +CREATE POLICY maintainer_select ON invite_maintainer_for_project FOR SELECT TO rsd_user + USING (project IN (SELECT * FROM projects_of_current_maintainer()) + OR created_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account') + OR claimed_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +CREATE POLICY maintainer_delete ON invite_maintainer_for_project FOR DELETE TO rsd_user + USING (project IN (SELECT * FROM projects_of_current_maintainer()) + OR created_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account') + OR claimed_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +CREATE POLICY maintainer_insert ON invite_maintainer_for_project FOR INSERT TO rsd_user + WITH CHECK (project IN (SELECT * FROM projects_of_current_maintainer()) AND created_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +CREATE POLICY admin_all_rights ON invite_maintainer_for_project TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + +-- invitations SOFTWARE +ALTER TABLE invite_maintainer_for_software ENABLE ROW LEVEL SECURITY; + +CREATE POLICY maintainer_select ON invite_maintainer_for_software FOR SELECT TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer()) + OR created_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account') + OR claimed_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +CREATE POLICY maintainer_delete ON invite_maintainer_for_software FOR DELETE TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer()) + OR created_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account') + OR claimed_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +CREATE POLICY maintainer_insert ON invite_maintainer_for_software FOR INSERT TO rsd_user + WITH CHECK (software IN (SELECT * FROM software_of_current_maintainer()) AND created_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +CREATE POLICY admin_all_rights ON invite_maintainer_for_software TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + +-- invitations ORGANISATION +ALTER TABLE invite_maintainer_for_organisation ENABLE ROW LEVEL SECURITY; + +CREATE POLICY maintainer_select ON invite_maintainer_for_organisation FOR SELECT TO rsd_user + USING (organisation IN (SELECT * FROM organisations_of_current_maintainer()) + OR created_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account') + OR claimed_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +CREATE POLICY maintainer_delete ON invite_maintainer_for_organisation FOR DELETE TO rsd_user + USING (organisation IN (SELECT * FROM organisations_of_current_maintainer()) + OR created_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account') + OR claimed_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +CREATE POLICY maintainer_insert ON invite_maintainer_for_organisation FOR INSERT TO rsd_user + WITH CHECK (organisation IN (SELECT * FROM organisations_of_current_maintainer()) AND created_by = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +CREATE POLICY admin_all_rights ON invite_maintainer_for_organisation TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- image +ALTER TABLE image ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON image FOR SELECT TO rsd_web_anon, rsd_user + USING (TRUE); + +CREATE POLICY rsd_user_all_rights ON image TO rsd_user + USING (TRUE) + WITH CHECK (TRUE); + +CREATE POLICY admin_all_rights ON image TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- software +ALTER TABLE software ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON software FOR SELECT TO rsd_web_anon, rsd_user + USING (is_published); + +CREATE POLICY maintainer_select_related ON software FOR SELECT TO rsd_user + USING (id IN (SELECT * FROM related_software())); + +CREATE POLICY maintainer_all_rights ON software TO rsd_user + USING (id IN (SELECT * FROM software_of_current_maintainer())) + WITH CHECK (TRUE); + +CREATE FUNCTION insert_maintainer_new_software() RETURNS TRIGGER LANGUAGE plpgsql SECURITY DEFINER AS +$$ +BEGIN + IF (SELECT current_setting('request.jwt.claims', TRUE)::json->>'account' IS NULL) THEN RETURN NULL; + END IF; + INSERT INTO maintainer_for_software VALUES (uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'), NEW.id); + RETURN NULL; +END +$$; + +CREATE TRIGGER insert_maintainer_new_software AFTER INSERT ON software FOR EACH ROW EXECUTE PROCEDURE insert_maintainer_new_software(); + +CREATE POLICY admin_all_rights ON software TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- software relations +ALTER TABLE repository_url ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON repository_url FOR SELECT TO rsd_web_anon, rsd_user + USING (software IN (SELECT id FROM software)); + +CREATE POLICY maintainer_all_rights ON repository_url TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer())) + WITH CHECK (software IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON repository_url TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE package_manager ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON package_manager FOR SELECT TO rsd_web_anon, rsd_user + USING (software IN (SELECT id FROM software)); + +CREATE POLICY maintainer_all_rights ON package_manager TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer())) + WITH CHECK (software IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON package_manager TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE license_for_software ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON license_for_software FOR SELECT TO rsd_web_anon, rsd_user + USING (software IN (SELECT id FROM software)); + +CREATE POLICY maintainer_all_rights ON license_for_software TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer())) + WITH CHECK (software IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON license_for_software TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE contributor ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON contributor FOR SELECT TO rsd_web_anon, rsd_user + USING (software IN (SELECT id FROM software)); + +CREATE POLICY maintainer_all_rights ON contributor TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer())) + WITH CHECK (software IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON contributor TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE testimonial ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON testimonial FOR SELECT TO rsd_web_anon, rsd_user + USING (software IN (SELECT id FROM software)); + +CREATE POLICY maintainer_all_rights ON testimonial TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer())) + WITH CHECK (software IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON testimonial TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- categories +ALTER TABLE category ENABLE ROW LEVEL SECURITY; + +-- allow everybody to read +CREATE POLICY anyone_can_read ON category + FOR SELECT + TO rsd_web_anon, rsd_user + USING (TRUE); + +CREATE POLICY maintainer_all_rights ON category + TO rsd_user + USING ( + (community IS NOT NULL AND community IN (SELECT * FROM communities_of_current_maintainer())) + OR + (organisation IS NOT NULL AND organisation IN (SELECT * FROM organisations_of_current_maintainer())) + ); + +-- allow admins to have full read/write access +CREATE POLICY admin_all_rights ON category + TO rsd_admin + USING (TRUE); + + +-- categories for software +ALTER TABLE category_for_software ENABLE ROW LEVEL SECURITY; + +-- allow everybody to read metadata of published software +CREATE POLICY anyone_can_read ON category_for_software + FOR SELECT + TO rsd_web_anon, rsd_user + USING (software_id IN (SELECT id FROM software)); + +-- allow software maintainers to have read/write access to their software +CREATE POLICY maintainer_all_rights ON category_for_software + TO rsd_user + USING (software_id IN (SELECT * FROM software_of_current_maintainer())); + +-- allow admins to have full read/write access +CREATE POLICY admin_all_rights ON category_for_software + TO rsd_admin + USING (TRUE); + + +-- categories for project +ALTER TABLE category_for_project ENABLE ROW LEVEL SECURITY; + +-- allow everybody to read metadata of published projects +CREATE POLICY anyone_can_read ON category_for_project + FOR SELECT + TO rsd_web_anon, rsd_user + USING (project_id IN (SELECT id FROM project)); + +-- allow software maintainers to have read/write access to their project +CREATE POLICY maintainer_all_rights ON category_for_project + TO rsd_user + USING (project_id IN (SELECT * FROM projects_of_current_maintainer())); + +-- allow admins to have full read/write access +CREATE POLICY admin_all_rights ON category_for_project + TO rsd_admin + USING (TRUE); + + +-- keywords +ALTER TABLE keyword ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON keyword FOR SELECT TO rsd_web_anon, rsd_user + USING (TRUE); + +CREATE POLICY maintainer_can_insert ON keyword FOR INSERT TO rsd_user + WITH CHECK (TRUE); + +CREATE POLICY maintainer_can_delete ON keyword FOR DELETE TO rsd_user + USING (TRUE); + +CREATE POLICY admin_all_rights ON keyword TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- keywords for software +ALTER TABLE keyword_for_software ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON keyword_for_software FOR SELECT TO rsd_web_anon, rsd_user + USING (software IN (SELECT id FROM software)); + +CREATE POLICY maintainer_all_rights ON keyword_for_software TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer())) + WITH CHECK (software IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON keyword_for_software TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- projects +ALTER TABLE project ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON project FOR SELECT TO rsd_web_anon, rsd_user + USING (is_published); + +CREATE POLICY maintainer_select_related ON project FOR SELECT TO rsd_user + USING (id IN (SELECT * FROM related_projects())); + +CREATE POLICY maintainer_all_rights ON project TO rsd_user + USING (id IN (SELECT * FROM projects_of_current_maintainer())) + WITH CHECK (TRUE); + +CREATE FUNCTION insert_maintainer_new_project() RETURNS TRIGGER LANGUAGE plpgsql SECURITY DEFINER AS +$$ +BEGIN + IF (SELECT current_setting('request.jwt.claims', TRUE)::json->>'account' IS NULL) THEN RETURN NULL; + END IF; + INSERT INTO maintainer_for_project VALUES (uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'), NEW.id); + RETURN NULL; +END +$$; + +CREATE TRIGGER insert_maintainer_new_project AFTER INSERT ON project FOR EACH ROW EXECUTE PROCEDURE insert_maintainer_new_project(); + +CREATE POLICY admin_all_rights ON project TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE url_for_project ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON url_for_project FOR SELECT TO rsd_web_anon, rsd_user + USING (project IN (SELECT id FROM project)); + +CREATE POLICY maintainer_all_rights ON url_for_project TO rsd_user + USING (project IN (SELECT * FROM projects_of_current_maintainer())) + WITH CHECK (project IN (SELECT * FROM projects_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON url_for_project TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + +-- project relations +ALTER TABLE team_member ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON team_member FOR SELECT TO rsd_web_anon, rsd_user + USING (project IN (SELECT id FROM project)); + +CREATE POLICY maintainer_all_rights ON team_member TO rsd_user + USING (project IN (SELECT * FROM projects_of_current_maintainer())) + WITH CHECK (project IN (SELECT * FROM projects_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON team_member TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- research domain +ALTER TABLE research_domain ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON research_domain FOR SELECT TO rsd_web_anon, rsd_user + USING (TRUE); + +CREATE POLICY admin_all_rights ON research_domain TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- community +-- RLS software_for_community table +ALTER TABLE software_for_community ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON software_for_community FOR SELECT TO rsd_web_anon, rsd_user + USING (software IN (SELECT id FROM software) AND status = 'approved'); + +CREATE POLICY maintainer_can_read ON software_for_community FOR SELECT TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer()) OR community IN (SELECT * FROM communities_of_current_maintainer())); + +CREATE POLICY maintainer_software_insert ON software_for_community FOR INSERT TO rsd_user + WITH CHECK (status = 'pending' AND software IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY maintainer_community_insert ON software_for_community FOR INSERT TO rsd_user + WITH CHECK (community IN (SELECT * FROM communities_of_current_maintainer())); + +CREATE POLICY maintainer_community_update ON software_for_community FOR UPDATE TO rsd_user + USING (community IN (SELECT * FROM communities_of_current_maintainer())); + +CREATE POLICY maintainer_software_delete ON software_for_community FOR DELETE TO rsd_user + USING ((status = 'pending' OR status = 'approved') AND software IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY maintainer_community_delete ON software_for_community FOR DELETE TO rsd_user + USING (community IN (SELECT * FROM communities_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON software_for_community TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- keywords and research domains for projects +ALTER TABLE keyword_for_project ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON keyword_for_project FOR SELECT TO rsd_web_anon, rsd_user + USING (project IN (SELECT id FROM project)); + +CREATE POLICY maintainer_all_rights ON keyword_for_project TO rsd_user + USING (project IN (SELECT * FROM projects_of_current_maintainer())) + WITH CHECK (project IN (SELECT * FROM projects_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON keyword_for_project TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE research_domain_for_project ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON research_domain_for_project FOR SELECT TO rsd_web_anon, rsd_user + USING (project IN (SELECT id FROM project)); + +CREATE POLICY maintainer_all_rights ON research_domain_for_project TO rsd_user + USING (project IN (SELECT * FROM projects_of_current_maintainer())) + WITH CHECK (project IN (SELECT * FROM projects_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON research_domain_for_project TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE keyword_for_community ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON keyword_for_community FOR SELECT TO rsd_web_anon, rsd_user + USING (TRUE); + +CREATE POLICY maintainer_insert ON keyword_for_community FOR INSERT TO rsd_user + WITH CHECK (community IN (SELECT * FROM communities_of_current_maintainer())); + +CREATE POLICY maintainer_delete ON keyword_for_community FOR DELETE TO rsd_user + USING (community IN (SELECT * FROM communities_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON keyword_for_community TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- mentions +ALTER TABLE mention ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON mention FOR SELECT TO rsd_web_anon, rsd_user + USING (TRUE); + +CREATE POLICY maintainer_can_delete ON mention FOR DELETE TO rsd_user + USING (TRUE); + +CREATE POLICY maintainer_can_insert ON mention FOR INSERT TO rsd_user + WITH CHECK (TRUE); + +CREATE POLICY admin_all_rights ON mention TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE mention_for_software ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON mention_for_software FOR SELECT TO rsd_web_anon, rsd_user + USING (software IN (SELECT id FROM software)); + +CREATE POLICY maintainer_all_rights ON mention_for_software TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer())) + WITH CHECK (software IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON mention_for_software TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE reference_paper_for_software ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON reference_paper_for_software FOR SELECT TO rsd_web_anon, rsd_user + USING (software IN (SELECT id FROM software)); + +CREATE POLICY maintainer_all_rights ON reference_paper_for_software TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer())) + WITH CHECK (software IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON reference_paper_for_software TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE citation_for_mention ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON citation_for_mention FOR SELECT TO rsd_web_anon, rsd_user + USING (TRUE); + +CREATE POLICY admin_all_rights ON citation_for_mention TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE output_for_project ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON output_for_project FOR SELECT TO rsd_web_anon, rsd_user + USING (project IN (SELECT id FROM project)); + +CREATE POLICY maintainer_all_rights ON output_for_project TO rsd_user + USING (project IN (SELECT * FROM projects_of_current_maintainer())) + WITH CHECK (project IN (SELECT * FROM projects_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON output_for_project TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE impact_for_project ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON impact_for_project FOR SELECT TO rsd_web_anon, rsd_user + USING (project IN (SELECT id FROM project)); + +CREATE POLICY maintainer_all_rights ON impact_for_project TO rsd_user + USING (project IN (SELECT * FROM projects_of_current_maintainer())) + WITH CHECK (project IN (SELECT * FROM projects_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON impact_for_project TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- releases +ALTER TABLE release ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON release FOR SELECT TO rsd_web_anon, rsd_user + USING (software IN (SELECT id FROM software)); + +CREATE POLICY maintainer_select ON release FOR SELECT TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON release TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE release_version ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON release_version FOR SELECT TO rsd_web_anon, rsd_user + USING (release_id IN (SELECT software FROM release)); + +CREATE POLICY maintainer_select ON release_version FOR SELECT TO rsd_user + USING (release_id IN (SELECT software FROM release)); + +CREATE POLICY admin_all_rights ON release_version TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- accounts +ALTER TABLE account ENABLE ROW LEVEL SECURITY; + +CREATE POLICY maintainer_all_rights ON account TO rsd_user + USING (id = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')) + WITH CHECK (id = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +CREATE POLICY admin_all_rights ON account TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE login_for_account ENABLE ROW LEVEL SECURITY; + +CREATE POLICY maintainer_select ON login_for_account FOR SELECT TO rsd_user + USING (account IN (SELECT id FROM account)); + +CREATE POLICY maintainer_delete ON login_for_account FOR DELETE TO rsd_user + USING (account IN (SELECT id FROM account)); + +CREATE POLICY admin_all_rights ON login_for_account TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE admin_account ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_all_rights ON admin_account TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + + +-- organisation +ALTER TABLE organisation ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON organisation FOR SELECT TO rsd_web_anon, rsd_user + USING (TRUE); + +CREATE POLICY maintainer_can_update ON organisation FOR UPDATE TO rsd_user + USING (id IN (SELECT * FROM organisations_of_current_maintainer())) + WITH CHECK (id IN (SELECT * FROM organisations_of_current_maintainer())); + +-- see the trigger sanitise_insert_organisation +CREATE POLICY maintainer_can_insert ON organisation FOR INSERT TO rsd_user + WITH CHECK (TRUE); + +CREATE POLICY admin_all_rights ON organisation TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + +-- inter relations +ALTER TABLE software_for_software ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON software_for_software FOR SELECT TO rsd_web_anon, rsd_user + USING (origin IN (SELECT id FROM software) AND relation IN (SELECT id FROM software)); + +CREATE POLICY maintainer_origin_can_read ON software_for_software FOR SELECT TO rsd_user + USING (origin IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY maintainer_origin_insert ON software_for_software FOR INSERT TO rsd_user + WITH CHECK (origin IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY maintainer_origin_delete ON software_for_software FOR DELETE TO rsd_user + USING (origin IN (SELECT * FROM software_of_current_maintainer())); + +CREATE POLICY admin_all_rights ON software_for_software TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE software_for_project ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON software_for_project FOR SELECT TO rsd_web_anon, rsd_user + USING (software IN (SELECT id FROM software) AND project IN (SELECT id FROM project)); + +CREATE POLICY maintainer_can_read ON software_for_project FOR SELECT TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer()) OR project IN (SELECT * FROM projects_of_current_maintainer())); + +CREATE POLICY maintainer_insert ON software_for_project FOR INSERT TO rsd_user + WITH CHECK (status = 'approved' AND (software IN (SELECT * FROM software_of_current_maintainer()) OR project IN (SELECT * FROM projects_of_current_maintainer()))); + +CREATE POLICY maintainer_update ON software_for_project FOR UPDATE TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer()) OR project IN (SELECT * FROM projects_of_current_maintainer())); + +CREATE POLICY maintainer_delete ON software_for_project FOR DELETE TO rsd_user + USING (status = 'approved' AND (software IN (SELECT * FROM software_of_current_maintainer()) OR project IN (SELECT * FROM projects_of_current_maintainer()))); + +CREATE POLICY admin_all_rights ON software_for_project TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE project_for_project ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON project_for_project FOR SELECT TO rsd_web_anon, rsd_user + USING (origin IN (SELECT id FROM project) AND relation IN (SELECT id FROM project)); + +CREATE POLICY maintainer_can_read ON project_for_project FOR SELECT TO rsd_user + USING (origin IN (SELECT * FROM projects_of_current_maintainer()) OR relation IN (SELECT * FROM projects_of_current_maintainer())); + +CREATE POLICY maintainer_insert ON project_for_project FOR INSERT TO rsd_user + WITH CHECK (status = 'approved' AND (origin IN (SELECT * FROM projects_of_current_maintainer()) OR relation IN (SELECT * FROM projects_of_current_maintainer()))); + +CREATE POLICY maintainer_update ON project_for_project FOR UPDATE TO rsd_user + USING (origin IN (SELECT * FROM projects_of_current_maintainer()) OR relation IN (SELECT * FROM projects_of_current_maintainer())); + +CREATE POLICY maintainer_delete ON project_for_project FOR DELETE TO rsd_user + USING (status = 'approved' AND (origin IN (SELECT * FROM projects_of_current_maintainer()) OR relation IN (SELECT * FROM projects_of_current_maintainer()))); + +CREATE POLICY admin_all_rights ON project_for_project TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE software_for_organisation ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON software_for_organisation FOR SELECT TO rsd_web_anon, rsd_user + USING (software IN (SELECT id FROM software)); + +CREATE POLICY maintainer_can_read ON software_for_organisation FOR SELECT TO rsd_user + USING (organisation IN (SELECT * FROM organisations_of_current_maintainer())); + +CREATE POLICY maintainer_insert ON software_for_organisation FOR INSERT TO rsd_user + WITH CHECK (status = 'approved' AND ((NOT is_featured AND software IN (SELECT * FROM software_of_current_maintainer())) OR organisation IN (SELECT * FROM organisations_of_current_maintainer()))); + +CREATE POLICY maintainer_update ON software_for_organisation FOR UPDATE TO rsd_user + USING (software IN (SELECT * FROM software_of_current_maintainer()) OR organisation IN (SELECT * FROM organisations_of_current_maintainer())); + +CREATE POLICY maintainer_delete ON software_for_organisation FOR DELETE TO rsd_user + USING (status = 'approved' AND (software IN (SELECT * FROM software_of_current_maintainer()) OR organisation IN (SELECT * FROM organisations_of_current_maintainer()))); + +CREATE POLICY admin_all_rights ON software_for_organisation TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +ALTER TABLE project_for_organisation ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON project_for_organisation FOR SELECT TO rsd_web_anon, rsd_user + USING (project IN (SELECT id FROM project)); + +CREATE POLICY maintainer_can_read ON project_for_organisation FOR SELECT TO rsd_user + USING (organisation IN (SELECT * FROM organisations_of_current_maintainer())); + +CREATE POLICY maintainer_insert ON project_for_organisation FOR INSERT TO rsd_user + WITH CHECK (status = 'approved' AND ((NOT is_featured AND project IN (SELECT * FROM projects_of_current_maintainer())) OR organisation IN (SELECT * FROM organisations_of_current_maintainer()))); + +CREATE POLICY maintainer_update ON project_for_organisation FOR UPDATE TO rsd_user + USING (project IN (SELECT * FROM projects_of_current_maintainer()) OR organisation IN (SELECT * FROM organisations_of_current_maintainer())); + +CREATE POLICY maintainer_delete ON project_for_organisation FOR DELETE TO rsd_user + USING (status = 'approved' AND (project IN (SELECT * FROM projects_of_current_maintainer()) OR organisation IN (SELECT * FROM organisations_of_current_maintainer()))); + +CREATE POLICY admin_all_rights ON project_for_organisation TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- meta-pages +ALTER TABLE meta_pages ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON meta_pages FOR SELECT TO rsd_web_anon, rsd_user + USING (TRUE); + +CREATE POLICY admin_all_rights ON meta_pages TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- backend logs +ALTER TABLE backend_log ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_all_rights ON backend_log TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- software_highlights +ALTER TABLE software_highlight ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON software_highlight FOR SELECT TO rsd_web_anon, rsd_user + USING (software IN (SELECT id FROM software)); + +CREATE POLICY admin_all_rights ON software_highlight TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); diff --git a/database/021-create-global-triggers.sql b/database/021-create-global-triggers.sql new file mode 100644 index 0000000..94ae599 --- /dev/null +++ b/database/021-create-global-triggers.sql @@ -0,0 +1,116 @@ +-- SPDX-FileCopyrightText: 2023 Christian Meeßen (GFZ) +-- SPDX-FileCopyrightText: 2023 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2023 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +-- SPDX-FileCopyrightText: 2023 Netherlands eScience Center +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- Checks whether the user agreed to the RSD terms and pricacy statement +CREATE FUNCTION check_user_agreement_on_action() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + IF + CURRENT_USER <> 'rsd_admin' AND NOT + (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) AND + (SELECT * FROM user_agreements_stored(uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'))) = FALSE + THEN + RAISE EXCEPTION USING MESSAGE = 'You need to agree to our Terms of Service and the Privacy Statement before proceeding. Please open your user profile settings to agree.'; + ELSE + RETURN NEW; + END IF; +END +$$; + +-- Triggers for UPDATE +CREATE PROCEDURE check_user_agreement_on_update_all_tables() LANGUAGE plpgsql AS +$$ +DECLARE + _sql VARCHAR; +BEGIN + FOR _sql IN SELECT CONCAT ( + 'CREATE TRIGGER check_', + quote_ident(table_name), + '_before_update BEFORE UPDATE ON ', + quote_ident(table_name), + ' FOR EACH STATEMENT EXECUTE FUNCTION check_user_agreement_on_action();' + ) + FROM + information_schema.tables + WHERE + table_schema = 'public' AND + table_name NOT IN ('account', 'login_for_account') + LOOP + EXECUTE _sql; + END LOOP; +END +$$; + +CALL check_user_agreement_on_update_all_tables(); + +-- Triggers for INSERT +CREATE PROCEDURE check_user_agreement_on_insert_all_tables() LANGUAGE plpgsql AS +$$ +DECLARE + _sql VARCHAR; +BEGIN + FOR _sql IN SELECT CONCAT ( + 'CREATE TRIGGER check_', + quote_ident(table_name), + '_before_insert BEFORE INSERT ON ', + quote_ident(table_name), + ' FOR EACH STATEMENT EXECUTE FUNCTION check_user_agreement_on_action();' + ) + FROM + information_schema.tables + WHERE + table_schema = 'public' AND + table_name NOT IN ('account', 'login_for_account') + LOOP + EXECUTE _sql; + END LOOP; +END +$$; + +CALL check_user_agreement_on_insert_all_tables(); + +-- Checks whether the user agreed to the RSD terms and pricacy statement +CREATE FUNCTION check_user_agreement_on_delete_action() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + IF + CURRENT_USER <> 'rsd_admin' AND NOT + (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) AND + (SELECT * FROM user_agreements_stored(uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'))) = FALSE + THEN + RAISE EXCEPTION USING MESSAGE = 'You need to agree to our Terms of Service and the Privacy Statement before proceeding. Please open your user profile settings to agree.'; + ELSE + RETURN OLD; + END IF; +END +$$; + +-- Triggers for DELETE +CREATE PROCEDURE check_user_agreement_on_delete_all_tables() LANGUAGE plpgsql AS +$$ +DECLARE + _sql VARCHAR; +BEGIN + FOR _sql IN SELECT CONCAT ( + 'CREATE TRIGGER check_', + quote_ident(table_name), + '_before_delete BEFORE DELETE ON ', + quote_ident(table_name), + ' FOR EACH STATEMENT EXECUTE FUNCTION check_user_agreement_on_delete_action();' + ) + FROM + information_schema.tables + WHERE + table_schema = 'public' AND + table_name NOT IN ('account', 'login_for_account') + LOOP + EXECUTE _sql; + END LOOP; +END +$$; + +CALL check_user_agreement_on_delete_all_tables(); diff --git a/database/022-create-maintenance-tables.sql b/database/022-create-maintenance-tables.sql new file mode 100644 index 0000000..b89dae0 --- /dev/null +++ b/database/022-create-maintenance-tables.sql @@ -0,0 +1,47 @@ +-- SPDX-FileCopyrightText: 2023 Christian Meeßen (GFZ) +-- SPDX-FileCopyrightText: 2023 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2023 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +-- SPDX-FileCopyrightText: 2023 Netherlands eScience Center +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE TABLE global_announcement ( +-- trick to only have one row in this table: + id BOOLEAN DEFAULT TRUE PRIMARY KEY CHECK (id), + text VARCHAR(2000), + enabled BOOLEAN DEFAULT FALSE NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE FUNCTION sanitise_insert_global_announcement () RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = TRUE; + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_global_announcement BEFORE INSERT ON global_announcement FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_global_announcement(); + +CREATE FUNCTION sanitise_update_global_announcement() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_global_announcement BEFORE UPDATE ON global_announcement FOR EACH ROW EXECUTE PROCEDURE sanitise_update_global_announcement(); + +ALTER TABLE global_announcement ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON global_announcement FOR SELECT TO rsd_web_anon, rsd_user USING (TRUE); + +CREATE POLICY admin_all_rights ON global_announcement TO rsd_admin USING (TRUE) WITH CHECK (TRUE); + +INSERT INTO global_announcement (id) VALUES (TRUE) diff --git a/database/023-news.sql b/database/023-news.sql new file mode 100644 index 0000000..16a5b1d --- /dev/null +++ b/database/023-news.sql @@ -0,0 +1,112 @@ +-- SPDX-FileCopyrightText: 2023 - 2024 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- NEWS table +CREATE TABLE news ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + slug VARCHAR(200) NOT NULL CHECK (slug ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + is_published BOOLEAN DEFAULT FALSE NOT NULL, + publication_date DATE NOT NULL, + title VARCHAR(200) NOT NULL, + author VARCHAR(200), + summary VARCHAR(300), + description VARCHAR(10000), + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +-- UNIQUE index on slug and publication date combination +CREATE UNIQUE INDEX unique_news_item ON news (slug,publication_date); + +-- SANITISE insert and update +CREATE FUNCTION sanitise_insert_news() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_news BEFORE INSERT ON news FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_news(); + + +CREATE FUNCTION sanitise_update_news() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + + IF NEW.slug IS DISTINCT FROM OLD.slug AND CURRENT_USER IS DISTINCT FROM 'rsd_admin' AND (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) IS DISTINCT FROM TRUE THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to change the slug'; + END IF; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_news BEFORE UPDATE ON news FOR EACH ROW EXECUTE PROCEDURE sanitise_update_news(); + +-- IMAGES FOR NEWS items +CREATE TABLE image_for_news ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + news UUID REFERENCES news (id) NOT NULL, + image_id VARCHAR(40) REFERENCES image(id) NOT NULL, + position VARCHAR(25) DEFAULT 'card', + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +-- UNIQUE index on news and image combination +CREATE UNIQUE INDEX unique_news_image ON image_for_news (news,image_id); + + +-- SANITISE insert and update +CREATE FUNCTION sanitise_insert_image_for_news() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_image_for_news BEFORE INSERT ON image_for_news FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_image_for_news(); + +CREATE FUNCTION sanitise_update_image_for_news() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_image_for_news BEFORE UPDATE ON image_for_news FOR EACH ROW EXECUTE PROCEDURE sanitise_update_image_for_news(); + + +-- RLS news pages +ALTER TABLE news ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read_published ON news FOR SELECT TO rsd_web_anon, rsd_user + USING (is_published); + +CREATE POLICY admin_all_rights ON news TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + +-- RLS image_for_news +ALTER TABLE image_for_news ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON image_for_news FOR SELECT TO rsd_web_anon, rsd_user + USING (news IN (SELECT id FROM news)); + +CREATE POLICY admin_all_rights ON image_for_news TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); diff --git a/database/024-remote-rsd.sql b/database/024-remote-rsd.sql new file mode 100644 index 0000000..3e60e7f --- /dev/null +++ b/database/024-remote-rsd.sql @@ -0,0 +1,89 @@ +-- SPDX-FileCopyrightText: 2024 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2024 Netherlands eScience Center +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- REMOTE_RSD +-- Table for remote rsd to scrape +CREATE TABLE remote_rsd ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + label VARCHAR (50) NOT NULL UNIQUE CHECK (LENGTH(label) >= 3), + domain VARCHAR(200) NOT NULL UNIQUE, + active BOOLEAN DEFAULT TRUE, + scrape_interval_minutes BIGINT DEFAULT 5 CHECK (scrape_interval_minutes >= 5), + scraped_at TIMESTAMPTZ, + last_err_msg VARCHAR(1000), + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +-- SANITIZE REMOTE_RSD +CREATE FUNCTION sanitise_insert_remote_rsd() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = gen_random_uuid(); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_remote_rsd BEFORE INSERT ON remote_rsd + FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_remote_rsd(); + +CREATE FUNCTION sanitise_update_remote_rsd() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.id = OLD.id; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_remote_rsd BEFORE UPDATE ON remote_rsd + FOR EACH ROW EXECUTE PROCEDURE sanitise_update_remote_rsd(); + +-- RLS REMOTE_RSD + +ALTER TABLE remote_rsd ENABLE ROW LEVEL SECURITY; + +CREATE POLICY anyone_can_read ON remote_rsd FOR SELECT TO rsd_web_anon, rsd_user + USING (TRUE); + +CREATE POLICY admin_all_rights ON remote_rsd TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- REMOTE_SOFTWARE +-- Table for scraped remote software +-- Results are returned from software_overview RPC from remote RSD and enriched with remote_rsd id +CREATE TABLE remote_software ( + id UUID DEFAULT gen_random_uuid() PRIMARY KEY, + remote_rsd_id UUID NOT NULL REFERENCES remote_rsd(id), + remote_software_id UUID NOT NULL, + slug VARCHAR(200) NOT NULL CHECK (slug ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + is_published BOOLEAN DEFAULT FALSE NOT NULL, + brand_name VARCHAR(200) NOT NULL, + short_statement VARCHAR(300), + image_id VARCHAR(40), + updated_at TIMESTAMPTZ, + contributor_cnt BIGINT, + mention_cnt BIGINT, + keywords CITEXT[], + keywords_text TEXT, + prog_lang TEXT[], + licenses VARCHAR[], + scraped_at TIMESTAMPTZ NOT NULL, + UNIQUE(remote_rsd_id, remote_software_id) +); + +CREATE POLICY anyone_can_read ON remote_software FOR SELECT TO rsd_web_anon, rsd_user + USING (is_published); + +ALTER TABLE remote_software ENABLE ROW LEVEL SECURITY; +CREATE POLICY admin_all_rights ON remote_software TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); diff --git a/database/025-rsd-info.sql b/database/025-rsd-info.sql new file mode 100644 index 0000000..61b88b8 --- /dev/null +++ b/database/025-rsd-info.sql @@ -0,0 +1,53 @@ +-- SPDX-FileCopyrightText: 2024 - 2025 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- RSD info table +-- used to obtain RSD name to use for remotes +-- it should provide basic info about rsd instance +-- manually insert remote_name property +CREATE TABLE rsd_info ( + key VARCHAR(100) PRIMARY KEY, + value VARCHAR(250) NOT NULL, + public BOOLEAN DEFAULT TRUE, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE FUNCTION sanitise_insert_rsd_info() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_rsd_info BEFORE INSERT ON + rsd_info FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_rsd_info(); + +CREATE FUNCTION sanitise_update_rsd_info() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_rsd_info BEFORE UPDATE ON + rsd_info FOR EACH ROW EXECUTE PROCEDURE sanitise_update_rsd_info(); + +-- RLS +-- rsd info table +ALTER TABLE rsd_info ENABLE ROW LEVEL SECURITY; + +-- anyone can read (SELECT) public keys +CREATE POLICY anyone_can_read ON rsd_info FOR SELECT TO rsd_web_anon, rsd_user + USING (public = TRUE); + +-- rsd_admin has all rights +CREATE POLICY admin_all_rights ON rsd_info TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); diff --git a/database/026-user-profile.sql b/database/026-user-profile.sql new file mode 100644 index 0000000..c946a4c --- /dev/null +++ b/database/026-user-profile.sql @@ -0,0 +1,143 @@ +-- SPDX-FileCopyrightText: 2025 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2025 Dusan Mijatovic (dv4all) (dv4all) +-- SPDX-FileCopyrightText: 2025 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2025 dv4all +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- USER PROFILE TABLE +CREATE TABLE user_profile ( + account UUID REFERENCES account (id) PRIMARY KEY, + given_names VARCHAR(200) NOT NULL, + family_names VARCHAR(200) NOT NULL, + email_address VARCHAR(200), + role VARCHAR(200) CHECK (role ~ '^\S+( \S+)*$'), + affiliation VARCHAR(200) CHECK (affiliation ~ '^\S+( \S+)*$'), + is_public BOOLEAN DEFAULT FALSE NOT NULL, + avatar_id VARCHAR(40) REFERENCES image(id), + description VARCHAR(10000), + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE FUNCTION sanitise_insert_user_profile() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + -- use account uuid from token ? + -- NEW.account = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'); + NEW.created_at = LOCALTIMESTAMP; + NEW.updated_at = NEW.created_at; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_insert_user_profile + BEFORE INSERT ON user_profile + FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_user_profile(); + + +CREATE FUNCTION sanitise_update_user_profile() RETURNS TRIGGER LANGUAGE plpgsql AS +$$ +BEGIN + NEW.account = OLD.account; + NEW.created_at = OLD.created_at; + NEW.updated_at = LOCALTIMESTAMP; + return NEW; +END +$$; + +CREATE TRIGGER sanitise_update_login_for_account + BEFORE UPDATE ON user_profile + FOR EACH ROW EXECUTE PROCEDURE sanitise_update_user_profile(); + +--------------------------------------------------------- +--------- ROW LEVEL SECURITY (RLS) ---------------------- +--------------------------------------------------------- +-- Step 1: Enable Row-Level Security +--------------------------------------- +ALTER TABLE user_profile ENABLE ROW LEVEL SECURITY; + +------------------------------------ +-- Step 2: Define RLS policies +------------------------------------ +-- RLS policy for rsd_web_anon (only public profiles) +CREATE POLICY public_user_profile ON user_profile FOR SELECT TO rsd_web_anon + USING (is_public); + +-- RLS policy for rsd_user (only your profile) +CREATE POLICY my_user_profile ON user_profile FOR SELECT TO rsd_user + USING (account = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +-- RLS INSERT policy for rsd_user +CREATE POLICY maintainer_insert_rights ON user_profile FOR INSERT TO rsd_user + WITH CHECK (account = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +-- RLS DELETE policy for rsd_user +CREATE POLICY maintainer_delete_rights ON user_profile FOR DELETE TO rsd_user + USING (account = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +-- RLS UPDATE policy for rsd_user +CREATE POLICY maintainer_update_rights ON user_profile FOR UPDATE TO rsd_user + USING (account = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')) + WITH CHECK (account = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account')); + +-- RLS policy for rsd_admin +CREATE POLICY admin_all_rights ON user_profile TO rsd_admin + USING (TRUE) + WITH CHECK (TRUE); + + +-- Create function with elevated privileges +-- in order to return only orcids for public profiles +-- USED by rpc/person_mentions in 104-person-views +CREATE FUNCTION public_profile() RETURNS TABLE ( + orcid VARCHAR +) LANGUAGE sql STABLE SECURITY DEFINER AS +$$ +SELECT + login_for_account.sub as orcid +FROM + login_for_account +INNER JOIN + user_profile ON login_for_account.account = user_profile.account +WHERE + login_for_account.provider='orcid' AND user_profile.is_public +$$; + +-- Public user profile information for profile page +-- this info overwrites aggregated contributors/team member info +-- from /rpc/person_mentions in 104-person-views +CREATE FUNCTION public_user_profile() RETURNS TABLE ( + display_name VARCHAR, + given_names VARCHAR, + family_names VARCHAR, + email_address VARCHAR, + affiliation VARCHAR, + "role" VARCHAR, + avatar_id VARCHAR(40), + orcid VARCHAR, + account UUID, + is_public BOOLEAN +) LANGUAGE sql STABLE SECURITY DEFINER AS +$$ +SELECT + (CONCAT(user_profile.given_names,' ',user_profile.family_names)) AS display_name, + user_profile.given_names, + user_profile.family_names, + user_profile.email_address, + user_profile.affiliation, + user_profile.role, + user_profile.avatar_id, + login_for_account.sub AS orcid, + user_profile.account, + user_profile.is_public +FROM + user_profile +LEFT JOIN + login_for_account ON + user_profile.account = login_for_account.account + AND + login_for_account.provider = 'orcid' +WHERE + user_profile.is_public +$$; diff --git a/database/100-create-api-views.sql b/database/100-create-api-views.sql new file mode 100644 index 0000000..4c1239d --- /dev/null +++ b/database/100-create-api-views.sql @@ -0,0 +1,926 @@ +-- SPDX-FileCopyrightText: 2021 - 2023 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2021 - 2023 dv4all +-- SPDX-FileCopyrightText: 2022 - 2023 Christian Meeßen (GFZ) +-- SPDX-FileCopyrightText: 2022 - 2023 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +-- SPDX-FileCopyrightText: 2022 - 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2022 - 2025 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2023 - 2025 Dusan Mijatovic (Netherlands eScience Center) +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- NOTE1: Moved views to (stable) functions because views do not allow for RLS (row-level-security) +-- SEE issue https://github.com/research-software-directory/RSD-as-a-service/issues/170 + +-- NOTE2: After creating new function you might need to reload postgREST to be able to access the function + +-- Keywords with the count used in software +-- used by search to show existing keywords with the count +CREATE FUNCTION keyword_count_for_software() RETURNS TABLE ( + id UUID, + keyword CITEXT, + cnt BIGINT +) LANGUAGE sql STABLE AS +$$ + SELECT + keyword.id, + keyword.value AS keyword, + keyword_count.cnt + FROM + keyword + LEFT JOIN + (SELECT + keyword_for_software.keyword, + COUNT(keyword_for_software.keyword) AS cnt + FROM + keyword_for_software + GROUP BY keyword_for_software.keyword + ) AS keyword_count ON keyword.id = keyword_count.keyword; +$$; + +-- Keywords by software +-- for selecting keywords of specific software +-- using filter ?software=eq.UUID +CREATE FUNCTION keywords_by_software() RETURNS TABLE ( + id UUID, + keyword CITEXT, + software UUID +) LANGUAGE sql STABLE AS +$$ + SELECT + keyword.id, + keyword.value AS keyword, + keyword_for_software.software + FROM + keyword_for_software + INNER JOIN + keyword ON keyword.id = keyword_for_software.keyword; +$$; + +-- Keywords grouped by software for filtering +-- We use array for selecting software with specific keywords +-- We use text value for "wild card" search +-- used by software_overview & software_search rpc +CREATE FUNCTION keyword_filter_for_software() RETURNS TABLE ( + software UUID, + keywords CITEXT[], + keywords_text TEXT +) LANGUAGE sql STABLE AS +$$ + SELECT + keyword_for_software.software AS software, + ARRAY_AGG( + keyword.value + ORDER BY value + ) AS keywords, + STRING_AGG( + keyword.value,' ' + ORDER BY value + ) AS keywords_text + FROM + keyword_for_software + INNER JOIN + keyword ON keyword.id = keyword_for_software.keyword + GROUP BY keyword_for_software.software; +$$; + +-- COUNT contributors per software +CREATE FUNCTION count_software_contributors() RETURNS TABLE (software UUID, contributor_cnt BIGINT) LANGUAGE sql STABLE AS +$$ +SELECT + software.id, COUNT(*) AS contributor_cnt + FROM + software + INNER JOIN + contributor ON contributor.software = software.id + GROUP BY + software.id; +$$; + +-- programming language filter for software +-- used by software_overview & software_search rpc +CREATE FUNCTION prog_lang_filter_for_software() RETURNS TABLE ( + software UUID, + prog_lang TEXT[] +) LANGUAGE sql STABLE AS +$$ + SELECT + repository_url.software, + (SELECT + ARRAY_AGG(p_lang ORDER BY repository_url.languages -> p_lang DESC) + FROM + JSONB_OBJECT_KEYS(repository_url.languages) p_lang + ) AS "prog_lang" + FROM + repository_url; +$$; + +-- license filter for software +-- used by software_overview & software_search rpc +CREATE FUNCTION license_filter_for_software() RETURNS TABLE ( + software UUID, + licenses VARCHAR[] +) LANGUAGE sql STABLE AS +$$ +SELECT + license_for_software.software, + ARRAY_AGG(license_for_software.license) +FROM + license_for_software +GROUP BY + license_for_software.software +; +$$; + +-- Software maintainer by software slug +CREATE FUNCTION maintainer_for_software_by_slug() RETURNS TABLE (maintainer UUID, software UUID, slug VARCHAR) LANGUAGE sql STABLE AS +$$ + SELECT + maintainer_for_software.maintainer, maintainer_for_software.software, software.slug + FROM + maintainer_for_software + LEFT JOIN + software ON software.id = maintainer_for_software.software; +$$; + +-- Participating organisations by software +-- requires software UUID +CREATE FUNCTION organisations_of_software(software_id UUID) RETURNS TABLE ( + id UUID, + slug VARCHAR, + primary_maintainer UUID, + name VARCHAR, + ror_id VARCHAR, + is_tenant BOOLEAN, + website VARCHAR, + rsd_path VARCHAR, + logo_id VARCHAR, + status relation_status, + "position" INTEGER, + software UUID +) LANGUAGE sql STABLE AS +$$ + SELECT + organisation.id AS id, + organisation.slug, + organisation.primary_maintainer, + organisation.name, + organisation.ror_id, + organisation.is_tenant, + organisation.website, + organisation_route.rsd_path, + organisation.logo_id, + software_for_organisation.status, + software_for_organisation.position, + software.id AS software + FROM + software + INNER JOIN + software_for_organisation ON software.id = software_for_organisation.software + INNER JOIN + organisation ON software_for_organisation.organisation = organisation.id + LEFT JOIN + organisation_route(organisation.id) ON organisation_route.organisation = organisation.id + WHERE + software.id = software_id; +$$; + +-- Software count by organisation +-- BY DEFAULT we return count of approved software +-- IF public is FALSE we return total count (as far as RLS allows) +CREATE FUNCTION software_count_by_organisation(public BOOLEAN DEFAULT TRUE) RETURNS TABLE ( + organisation UUID, + software_cnt BIGINT +) LANGUAGE sql STABLE AS +$$ +SELECT + list_parent_organisations.organisation_id, + COUNT(DISTINCT software_for_organisation.software) AS software_cnt +FROM + software_for_organisation +CROSS JOIN list_parent_organisations(software_for_organisation.organisation) +WHERE + (NOT public) + OR + ( + software_for_organisation.status = 'approved' + AND + software IN (SELECT id FROM software WHERE is_published) + ) +GROUP BY list_parent_organisations.organisation_id; +$$; + +-- Project count by organisation +-- BY DEFAULT we return count of approved projects +-- IF public is FALSE we return total count (as far as RLS allows) +CREATE FUNCTION project_count_by_organisation(public BOOLEAN DEFAULT TRUE) RETURNS TABLE ( + organisation UUID, + project_cnt BIGINT +)LANGUAGE sql STABLE AS +$$ +SELECT + list_parent_organisations.organisation_id, + COUNT(DISTINCT project_for_organisation.project) AS project_cnt +FROM + project_for_organisation +CROSS JOIN list_parent_organisations(project_for_organisation.organisation) +WHERE + (NOT public) + OR + ( + status = 'approved' + AND + project IN (SELECT id FROM project WHERE is_published) + ) +GROUP BY list_parent_organisations.organisation_id; +$$; + +-- Direct children count by organisation +CREATE FUNCTION children_count_by_organisation() RETURNS TABLE ( + parent UUID, + children_cnt BIGINT +)LANGUAGE sql STABLE AS +$$ +SELECT + organisation.parent, COUNT(*) AS children_cnt +FROM + organisation +WHERE + organisation.parent IS NOT NULL +GROUP BY + organisation.parent +; +$$; + +-- Software releases by organisation +-- release info is scraped from concept DOI +-- one software belongs to multiple organisations +-- INCLUDES releases of children organisations +CREATE FUNCTION releases_by_organisation() RETURNS TABLE ( + organisation_id UUID, + software_id UUID, + software_slug VARCHAR, + software_name VARCHAR, + release_doi CITEXT, + release_tag VARCHAR, + release_date TIMESTAMPTZ, + release_year SMALLINT, + release_authors VARCHAR +) LANGUAGE sql STABLE AS +$$ +SELECT DISTINCT ON (organisation_id, software_id, mention.id) + organisation.id AS organisation_id, + software.id AS software_id, + software.slug AS software_slug, + software.brand_name AS software_name, + mention.doi AS release_doi, + mention.version AS release_tag, + mention.doi_registration_date AS release_date, + mention.publication_year AS release_year, + mention.authors AS release_authors +FROM + organisation +CROSS JOIN + list_child_organisations(organisation.id) +INNER JOIN + software_for_organisation ON list_child_organisations.organisation_id = software_for_organisation.organisation +INNER JOIN + software ON software.id = software_for_organisation.software +INNER JOIN + "release" ON "release".software = software.id +INNER JOIN + release_version ON release_version.release_id = "release".software +INNER JOIN + mention ON mention.id = release_version.mention_id +; +$$; + +-- Software releases count by organisation +-- DEPENDS ON releases_by_organisation RPC +CREATE FUNCTION release_cnt_by_organisation() RETURNS TABLE ( + organisation_id UUID, + release_cnt BIGINT +) LANGUAGE sql STABLE AS +$$ +SELECT + releases_by_organisation.organisation_id AS organisation_id, + COUNT(releases_by_organisation.*) AS release_cnt +FROM + organisation +INNER JOIN + releases_by_organisation() ON releases_by_organisation.organisation_id = organisation.id +GROUP BY + releases_by_organisation.organisation_id +; +$$; + +-- Software releases count per YEAR by organisation +-- DEPENDS ON releases_by_organisation RPC +CREATE FUNCTION release_cnt_by_year(organisation_id UUID) RETURNS TABLE ( + release_year SMALLINT, + release_cnt BIGINT +) LANGUAGE sql STABLE AS +$$ + SELECT + releases_by_organisation.release_year, + COUNT(releases_by_organisation.*) AS release_cnt + FROM + releases_by_organisation() + WHERE + releases_by_organisation.organisation_id = release_cnt_by_year.organisation_id + GROUP BY + releases_by_organisation.release_year; +$$; + +-- Organisations overview +-- we pass public param to count functions to get public/private count +-- public count is default, +-- note! the RLS will limit row selection in any case +CREATE FUNCTION organisations_overview(public BOOLEAN DEFAULT TRUE) RETURNS TABLE ( + id UUID, + slug VARCHAR, + parent UUID, + primary_maintainer UUID, + name VARCHAR, + short_description VARCHAR, + country VARCHAR, + ror_id VARCHAR, + website VARCHAR, + is_tenant BOOLEAN, + ror_names_string VARCHAR, + rsd_path VARCHAR, + parent_names VARCHAR, + logo_id VARCHAR, + software_cnt BIGINT, + project_cnt BIGINT, + children_cnt BIGINT, + release_cnt BIGINT, + score BIGINT +) LANGUAGE sql STABLE AS +$$ +SELECT + organisation.id, + organisation.slug, + organisation.parent, + organisation.primary_maintainer, + organisation.name, + organisation.short_description, + organisation.country, + organisation.ror_id, + organisation.website, + organisation.is_tenant, + organisation.ror_names_string, + organisation_route.rsd_path, + organisation_route.parent_names, + organisation.logo_id, + software_count_by_organisation.software_cnt, + project_count_by_organisation.project_cnt, + children_count_by_organisation.children_cnt, + release_cnt_by_organisation.release_cnt, + ( + COALESCE(software_count_by_organisation.software_cnt,0) + + COALESCE(project_count_by_organisation.project_cnt,0) + ) as score +FROM + organisation +LEFT JOIN + organisation_route(organisation.id) ON organisation_route.organisation = organisation.id +LEFT JOIN + software_count_by_organisation(public) ON software_count_by_organisation.organisation = organisation.id +LEFT JOIN + project_count_by_organisation(public) ON project_count_by_organisation.organisation = organisation.id +LEFT JOIN + children_count_by_organisation() ON children_count_by_organisation.parent = organisation.id +LEFT JOIN + release_cnt_by_organisation() ON release_cnt_by_organisation.organisation_id = organisation.id +; +$$; + +-- Participating organisations by project +-- we filter this view by project_id UUID +CREATE FUNCTION organisations_of_project(project_id UUID) RETURNS TABLE ( + id UUID, + slug VARCHAR, + primary_maintainer UUID, + name VARCHAR, + ror_id VARCHAR, + is_tenant BOOLEAN, + website VARCHAR, + rsd_path VARCHAR, + logo_id VARCHAR, + status relation_status, + role organisation_role, + "position" INTEGER, + project UUID, + parent UUID +) LANGUAGE sql STABLE AS +$$ + SELECT + organisation.id AS id, + organisation.slug, + organisation.primary_maintainer, + organisation.name, + organisation.ror_id, + organisation.is_tenant, + organisation.website, + organisation_route.rsd_path, + organisation.logo_id, + project_for_organisation.status, + project_for_organisation.role, + project_for_organisation.position, + project.id AS project, + organisation.parent + FROM + project + INNER JOIN + project_for_organisation ON project.id = project_for_organisation.project + INNER JOIN + organisation ON project_for_organisation.organisation = organisation.id + LEFT JOIN + organisation_route(organisation.id) ON organisation_route.organisation = organisation.id + WHERE + project.id = project_id; +$$; + +-- Project maintainer by slug +-- there are similar functions for software maintainers +CREATE FUNCTION maintainer_for_project_by_slug() RETURNS TABLE ( + maintainer UUID, + project UUID, + slug VARCHAR +) LANGUAGE sql STABLE AS +$$ + SELECT + maintainer_for_project.maintainer, + maintainer_for_project.project, + project.slug + FROM + maintainer_for_project + LEFT JOIN + project ON project.id = maintainer_for_project.project; +$$; + +-- Project maintainers list with basic personal info +-- used in the project maintainer list +CREATE FUNCTION maintainers_of_project(project_id UUID) RETURNS TABLE ( + maintainer UUID, + name VARCHAR[], + email VARCHAR[], + affiliation VARCHAR[] +) LANGUAGE plpgsql STABLE SECURITY DEFINER AS +$$ +DECLARE account_authenticated UUID; +BEGIN + account_authenticated = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'); + IF account_authenticated IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please login first'; + END IF; + + IF project_id IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please provide a project id'; + END IF; + + IF NOT project_id IN (SELECT * FROM projects_of_current_maintainer()) AND + CURRENT_USER IS DISTINCT FROM 'rsd_admin' AND ( + SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER + ) IS DISTINCT FROM TRUE THEN + RAISE EXCEPTION USING MESSAGE = 'You are not a maintainer of this project'; + END IF; + + RETURN QUERY + SELECT + maintainer_for_project.maintainer, + ARRAY_AGG(login_for_account.name), + ARRAY_AGG(login_for_account.email), + ARRAY_AGG(login_for_account.home_organisation) AS affiliation + FROM + maintainer_for_project + JOIN + login_for_account ON maintainer_for_project.maintainer = login_for_account.account + WHERE maintainer_for_project.project = project_id + GROUP BY maintainer_for_project.maintainer; + RETURN; +END +$$; + +-- Keywords with the count used in projects +-- used by search to show existing keywords with the count +CREATE FUNCTION keyword_count_for_projects() RETURNS TABLE ( + id UUID, + keyword CITEXT, + cnt BIGINT +) LANGUAGE sql STABLE AS +$$ + SELECT + keyword.id, + keyword.value AS keyword, + keyword_count.cnt + FROM + keyword + LEFT JOIN + (SELECT + keyword_for_project.keyword, + COUNT(keyword_for_project.keyword) AS cnt + FROM + keyword_for_project + GROUP BY keyword_for_project.keyword + ) AS keyword_count ON keyword.id = keyword_count.keyword; +$$; + +-- Keywords by project +-- for selecting keywords of specific project +-- using filter ?project=eq.UUID +CREATE FUNCTION keywords_by_project() RETURNS TABLE ( + id UUID, + keyword CITEXT, + project UUID +) LANGUAGE sql STABLE AS +$$ + SELECT + keyword.id, + keyword.value AS keyword, + keyword_for_project.project + FROM + keyword_for_project + INNER JOIN + keyword ON keyword.id = keyword_for_project.keyword; +$$; + + +-- Research domains by project +CREATE FUNCTION research_domain_by_project() RETURNS TABLE ( + id UUID, + "key" VARCHAR, + name VARCHAR, + description VARCHAR, + project UUID +) LANGUAGE sql STABLE AS +$$ + SELECT + research_domain.id, + research_domain.key, + research_domain.name, + research_domain.description, + research_domain_for_project.project + FROM + research_domain_for_project + INNER JOIN + research_domain ON research_domain.id=research_domain_for_project.research_domain; +$$; + +-- Software maintainers list with basic personal info +-- used in the software maintainer list +CREATE FUNCTION maintainers_of_software(software_id UUID) RETURNS TABLE ( + maintainer UUID, + name VARCHAR[], + email VARCHAR[], + affiliation VARCHAR[] +) LANGUAGE plpgsql STABLE SECURITY DEFINER AS +$$ +DECLARE account_authenticated UUID; +BEGIN + account_authenticated = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'); + IF account_authenticated IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please login first'; + END IF; + + IF software_id IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please provide a software id'; + END IF; + + IF NOT software_id IN (SELECT * FROM software_of_current_maintainer()) AND + CURRENT_USER IS DISTINCT FROM 'rsd_admin' AND ( + SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER + ) IS DISTINCT FROM TRUE THEN + RAISE EXCEPTION USING MESSAGE = 'You are not a maintainer of this software'; + END IF; + + RETURN QUERY + SELECT + maintainer_for_software.maintainer, + ARRAY_AGG(login_for_account.name), + ARRAY_AGG(login_for_account.email), + ARRAY_AGG(login_for_account.home_organisation) AS affiliation + FROM + maintainer_for_software + JOIN + login_for_account ON maintainer_for_software.maintainer = login_for_account.account + WHERE maintainer_for_software.software = software_id + GROUP BY maintainer_for_software.maintainer; + RETURN; +END +$$; + + +-- ORGANISATIONS BY MAINTAINER +-- NOTE! each organisation is shown multiple times in this view +-- we filter this view at least by user account (maintainer_id uuid) on primary_maintainer or maintainer +CREATE FUNCTION organisations_by_maintainer(maintainer_id UUID) RETURNS TABLE ( + id UUID, + slug VARCHAR, + parent UUID, + primary_maintainer UUID, + name VARCHAR, + short_description VARCHAR, + ror_id VARCHAR, + website VARCHAR, + is_tenant BOOLEAN, + logo_id VARCHAR, + software_cnt BIGINT, + project_cnt BIGINT, + children_cnt BIGINT, + rsd_path VARCHAR +) LANGUAGE sql STABLE AS +$$ +SELECT DISTINCT ON (organisation.id) + organisation.id, + organisation.slug, + organisation.parent, + organisation.primary_maintainer, + organisation.name, + organisation.short_description, + organisation.ror_id, + organisation.website, + organisation.is_tenant, + organisation.logo_id, + software_count_by_organisation.software_cnt, + project_count_by_organisation.project_cnt, + children_count_by_organisation.children_cnt, + organisation_route.rsd_path +FROM + organisation +LEFT JOIN + software_count_by_organisation() ON software_count_by_organisation.organisation = organisation.id +LEFT JOIN + project_count_by_organisation() ON project_count_by_organisation.organisation = organisation.id +LEFT JOIN + children_count_by_organisation() ON children_count_by_organisation.parent = organisation.id +LEFT JOIN + maintainer_for_organisation ON maintainer_for_organisation.organisation = organisation.id +LEFT JOIN + organisation_route(organisation.id) ON organisation_route.organisation = organisation.id +WHERE + maintainer_for_organisation.maintainer = maintainer_id OR organisation.primary_maintainer = maintainer_id; +$$; + +-- COUNTS by maintainer +-- software_cnt, project_cnt, organisation_cnt, communities_cnt +-- counts for user profile pages +-- this rpc returns json object instead of array +CREATE FUNCTION counts_by_maintainer( + OUT software_cnt BIGINT, + OUT project_cnt BIGINT, + OUT organisation_cnt BIGINT, + OUT community_cnt BIGINT +) LANGUAGE plpgsql STABLE AS +$$ +BEGIN + SELECT COUNT(*) FROM software_of_current_maintainer() INTO software_cnt; + SELECT COUNT(*) FROM projects_of_current_maintainer() INTO project_cnt; + SELECT COUNT(DISTINCT organisations_of_current_maintainer) + FROM organisations_of_current_maintainer() INTO organisation_cnt; + SELECT COUNT(*) FROM communities_of_current_maintainer() INTO community_cnt; +END +$$; + + +-- ORGANISATION maintainers list with basic personal info +-- used in the organisation maintainers page +CREATE FUNCTION maintainers_of_organisation(organisation_id UUID) RETURNS TABLE ( + maintainer UUID, + name VARCHAR[], + email VARCHAR[], + affiliation VARCHAR[], + is_primary BOOLEAN +) LANGUAGE plpgsql STABLE SECURITY DEFINER AS +$$ +DECLARE account_authenticated UUID; +BEGIN + account_authenticated = uuid(current_setting('request.jwt.claims', FALSE)::json->>'account'); + IF account_authenticated IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please login first'; + END IF; + + IF organisation_id IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please provide a organisation id'; + END IF; + + IF NOT organisation_id IN (SELECT * FROM organisations_of_current_maintainer()) AND + CURRENT_USER IS DISTINCT FROM 'rsd_admin' AND ( + SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER + ) IS DISTINCT FROM TRUE THEN + RAISE EXCEPTION USING MESSAGE = 'You are not a maintainer of this organisation'; + END IF; + + RETURN QUERY + WITH maintainer_ids AS ( + -- primary maintainer of organisation + SELECT + organisation.primary_maintainer AS maintainer, + TRUE AS is_primary + FROM + organisation + WHERE + organisation.id = organisation_id + -- append second selection + UNION ALL + -- other maintainers of organisation + SELECT + maintainer_for_organisation.maintainer AS maintainer, + FALSE AS is_primary + FROM + maintainer_for_organisation + WHERE + maintainer_for_organisation.organisation = organisation_id + ) + SELECT + maintainer_ids.maintainer AS maintainer, + ARRAY_AGG(login_for_account."name") AS name, + ARRAY_AGG(login_for_account.email) AS email, + ARRAY_AGG(login_for_account.home_organisation) AS affiliation, + BOOL_OR(maintainer_ids.is_primary) AS is_primary + FROM + maintainer_ids + INNER JOIN + login_for_account ON login_for_account.account = maintainer_ids.maintainer + GROUP BY + maintainer_ids.maintainer + -- primary as first record + ORDER BY + is_primary DESC; + RETURN; +END +$$; + +-- TOTAL COUNTS FOR HOMEPAGE +-- software_cnt, project_cnt, organisation_cnt +-- this rpc returns json object instead of array +-- DEPENDS on RPC mentions_by_software (104-software-views.sql) +CREATE FUNCTION homepage_counts( + OUT software_cnt BIGINT, + OUT open_software_cnt BIGINT, + OUT project_cnt BIGINT, + OUT organisation_cnt BIGINT, + OUT contributor_cnt BIGINT, + OUT software_mention_cnt BIGINT +) LANGUAGE plpgsql STABLE AS +$$ +BEGIN + SELECT COUNT(*) FROM software INTO software_cnt; + SELECT COUNT(*) FROM software WHERE NOT closed_source INTO open_software_cnt; + SELECT COUNT(*) FROM project INTO project_cnt; + SELECT + COUNT(*) AS organisation_cnt + FROM + organisations_overview(TRUE) + WHERE + organisations_overview.parent IS NULL AND organisations_overview.score>0 + INTO organisation_cnt; + SELECT COUNT(DISTINCT(orcid,given_names,family_names)) FROM contributor INTO contributor_cnt; + SELECT COUNT(*) FROM mentions_by_software() INTO software_mention_cnt; +END +$$; + + +-- Keywords grouped by project +-- We use keywords array for filtering +-- We use keywords_text for "wild card" search +CREATE FUNCTION keyword_filter_for_project() RETURNS TABLE ( + project UUID, + keywords CITEXT[], + keywords_text TEXT +) LANGUAGE sql STABLE AS +$$ + SELECT + keyword_for_project.project AS project, + ARRAY_AGG( + keyword.value + ORDER BY value + ) AS keywords, + STRING_AGG( + keyword.value,' ' + ORDER BY value + ) AS keywords_text + FROM + keyword_for_project + INNER JOIN + keyword ON keyword.id = keyword_for_project.keyword + GROUP BY keyword_for_project.project; +$$; + +-- RESEARCH DOMAIN grouped by project +-- We use array for exact filtering by KEY +-- We use research_domain_text for "wild card" search when filter is not active +CREATE FUNCTION research_domain_filter_for_project() RETURNS TABLE ( + project UUID, + research_domain VARCHAR[], + research_domain_text TEXT +) LANGUAGE sql STABLE AS +$$ + SELECT + research_domain_for_project.project AS project, + ARRAY_AGG( + research_domain.key + ORDER BY key + ) AS research_domain, + STRING_AGG( + research_domain.key || ' ' || research_domain."name",' ' + ORDER BY key + ) AS research_domain_text + FROM + research_domain_for_project + INNER JOIN + research_domain ON research_domain.id = research_domain_for_project.research_domain + GROUP BY research_domain_for_project.project; +$$; + +-- Check whether user agreed on Terms of Service and read the Privacy Statement +CREATE FUNCTION user_agreements_stored(account_id UUID) RETURNS BOOLEAN LANGUAGE sql STABLE AS +$$ + SELECT ( + account.agree_terms AND + account.notice_privacy_statement + ) + FROM + account + WHERE + account.id = account_id; +$$; + +-- Display amount of users per home_organisation +CREATE VIEW user_count_per_home_organisation AS + SELECT + home_organisation, + COUNT(*) + FROM + login_for_account + GROUP BY + home_organisation + ; + +-- Return the number of accounts since specified time stamp +CREATE FUNCTION new_accounts_count_since_timestamp(timestmp TIMESTAMPTZ) RETURNS INTEGER +LANGUAGE sql SECURITY DEFINER STABLE AS +$$ +SELECT + COUNT(account.created_at) +FROM + account +WHERE + created_at > timestmp; +$$; + +-- Keywords use by software and projects +-- DEPENDS ON FUNCTIONS keyword_count_for_software and keyword_count_for_projects +CREATE FUNCTION keyword_cnt() RETURNS TABLE ( + id UUID, + keyword CITEXT, + software_cnt BIGINT, + projects_cnt BIGINT +) LANGUAGE sql SECURITY DEFINER STABLE AS +$$ +SELECT + keyword.id, + keyword.value AS keyword, + keyword_count_for_software.cnt AS software_cnt, + keyword_count_for_projects.cnt AS projects_cnt +FROM + keyword +LEFT JOIN + keyword_count_for_software() ON keyword.value = keyword_count_for_software.keyword +LEFT JOIN + keyword_count_for_projects() ON keyword.value = keyword_count_for_projects.keyword +; +$$; + +-- Suggest a platform type based on the input of other users in the RSD +CREATE FUNCTION suggest_platform(hostname VARCHAR(200)) RETURNS platform_type +LANGUAGE SQL STABLE AS +$$ +SELECT + code_platform +FROM + ( + SELECT + url, + code_platform + FROM + repository_url + ) AS sub +WHERE + ( + -- Returns the hostname of sub.url + SELECT + TOKEN + FROM + ts_debug(sub.url) + WHERE + alias = 'host' + ) = hostname +GROUP BY + sub.code_platform +ORDER BY + COUNT(*) +DESC LIMIT + 1; +; +$$; diff --git a/database/102-create-account-functions.sql b/database/102-create-account-functions.sql new file mode 100644 index 0000000..9d928db --- /dev/null +++ b/database/102-create-account-functions.sql @@ -0,0 +1,45 @@ +-- SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2025 Dusan Mijatovic (Netherlands eScience Center) +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE FUNCTION delete_account(account_id UUID) RETURNS VOID LANGUAGE plpgsql VOLATILE AS +$$ +DECLARE account_authenticated UUID; +BEGIN + IF + account_id IS NULL + THEN + RAISE EXCEPTION USING MESSAGE = 'Please provide an account id'; + END IF; + account_authenticated = uuid(current_setting('request.jwt.claims', TRUE)::json->>'account'); + IF + CURRENT_USER IS DISTINCT FROM 'rsd_admin' + AND + (SELECT rolsuper FROM pg_roles WHERE rolname = CURRENT_USER) IS DISTINCT FROM TRUE + AND + ( + account_authenticated IS NULL OR account_authenticated IS DISTINCT FROM account_id + ) + THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to delete this account'; + END IF; + DELETE FROM maintainer_for_software WHERE maintainer = account_id; + DELETE FROM maintainer_for_project WHERE maintainer = account_id; + DELETE FROM maintainer_for_organisation WHERE maintainer = account_id; + DELETE FROM maintainer_for_community WHERE maintainer = account_id; + DELETE FROM invite_maintainer_for_software WHERE created_by = account_id OR claimed_by = account_id; + DELETE FROM invite_maintainer_for_project WHERE created_by = account_id OR claimed_by = account_id; + DELETE FROM invite_maintainer_for_organisation WHERE created_by = account_id OR claimed_by = account_id; + DELETE FROM invite_maintainer_for_community WHERE created_by = account_id OR claimed_by = account_id; + UPDATE organisation SET primary_maintainer = NULL WHERE primary_maintainer = account_id; + UPDATE community SET primary_maintainer = NULL WHERE primary_maintainer = account_id; + UPDATE contributor SET account = NULL WHERE account = account_id; + UPDATE team_member SET account = NULL WHERE account = account_id; + DELETE FROM admin_account WHERE admin_account.account_id = delete_account.account_id; + DELETE FROM login_for_account WHERE account = account_id; + DELETE FROM user_profile WHERE account = account_id; + DELETE FROM account WHERE id = account_id; +END +$$; diff --git a/database/104-person-views.sql b/database/104-person-views.sql new file mode 100644 index 0000000..d8fd49b --- /dev/null +++ b/database/104-person-views.sql @@ -0,0 +1,186 @@ +-- SPDX-FileCopyrightText: 2023 - 2025 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2023 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2023 dv4all +-- SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- ALL unique persons in RSD (contributors and team members) with their roles +-- TO BE USED IN SEARCH from 23-3-2023 +-- NOTE! UNION takes care of duplicate entries +CREATE FUNCTION unique_person_entries() RETURNS TABLE ( + display_name TEXT, + given_names VARCHAR, + family_names VARCHAR, + email_address VARCHAR, + affiliation VARCHAR, + "role" VARCHAR, + avatar_id VARCHAR, + orcid VARCHAR, + account UUID +) LANGUAGE sql STABLE AS +$$ +SELECT DISTINCT + (CONCAT(contributor.given_names,' ',contributor.family_names)) AS display_name, + contributor.given_names, + contributor.family_names, + contributor.email_address, + contributor.affiliation, + contributor.role, + contributor.avatar_id, + contributor.orcid, + contributor.account +FROM + contributor +UNION +SELECT DISTINCT + (CONCAT(team_member.given_names,' ',team_member.family_names)) AS display_name, + team_member.given_names, + team_member.family_names, + team_member.email_address, + team_member.affiliation, + team_member.role, + team_member.avatar_id, + team_member.orcid, + team_member.account +FROM + team_member +ORDER BY + display_name ASC; +$$; + +-- Aggregate avatars by display_name (given_names + family_names) +-- USED in rsd admins section +CREATE FUNCTION person_avatars_by_name() RETURNS TABLE ( + display_name TEXT, + avatars VARCHAR[] +) LANGUAGE sql STABLE AS +$$ + SELECT + unique_person_entries.display_name, + array_agg(DISTINCT(unique_person_entries.avatar_id)) AS avatars + FROM + unique_person_entries() + WHERE + unique_person_entries.avatar_id IS NOT NULL + GROUP BY + unique_person_entries.display_name + ; +$$; + +-- Aggregate avatars by orcid (given_names + family_names) +-- USED in rsd admins section +CREATE FUNCTION person_avatars_by_orcid() RETURNS TABLE ( + orcid TEXT, + avatars VARCHAR[] +) LANGUAGE sql STABLE AS +$$ + SELECT + unique_person_entries.orcid, + array_agg(DISTINCT(unique_person_entries.avatar_id)) AS avatars + FROM + unique_person_entries() + WHERE + unique_person_entries.avatar_id IS NOT NULL AND + unique_person_entries.orcid IS NOT NULL + GROUP BY + unique_person_entries.orcid + ; +$$; + +-- LIST ALL persons mentioned in software and projects +-- origin indicates source table +-- slug is direct link to edit this entry +-- avatars are aggregated on display_name (given_names + ' ' + family_names) +-- AND avatars are aggregated on ORCID +CREATE FUNCTION person_mentions() RETURNS TABLE ( + id UUID, + given_names VARCHAR, + family_names VARCHAR, + email_address VARCHAR, + affiliation VARCHAR, + "role" VARCHAR, + orcid VARCHAR, + avatar_id VARCHAR, + origin VARCHAR, + slug VARCHAR, + public_orcid_profile VARCHAR, + avatars_by_name VARCHAR[], + avatars_by_orcid VARCHAR[] +) LANGUAGE sql STABLE AS +$$ +SELECT + contributor.id, + contributor.given_names, + contributor.family_names, + contributor.email_address, + contributor.affiliation, + contributor.role, + contributor.orcid, + contributor.avatar_id, + 'contributor' AS origin, + software.slug, + public_profile.orcid as public_orcid_profile, + person_avatars_by_name.avatars AS avatars_by_name, + person_avatars_by_orcid.avatars AS avatars_by_orcid +FROM + contributor +INNER JOIN + software ON contributor.software = software.id +LEFT JOIN + public_profile() ON public_profile.orcid=contributor.orcid +LEFT JOIN + person_avatars_by_name() ON person_avatars_by_name.display_name = CONCAT(contributor.given_names,' ',contributor.family_names) +LEFT JOIN + person_avatars_by_orcid() ON person_avatars_by_orcid.orcid = contributor.orcid +UNION +SELECT + team_member.id, + team_member.given_names, + team_member.family_names, + team_member.email_address, + team_member.affiliation, + team_member.role, + team_member.orcid, + team_member.avatar_id, + 'team_member' AS origin, + project.slug, + public_profile.orcid as public_orcid_profile, + person_avatars_by_name.avatars AS avatars_by_name, + person_avatars_by_orcid.avatars AS avatars_by_orcid +FROM + team_member +INNER JOIN + project ON team_member.project = project.id +LEFT JOIN + public_profile() ON public_profile.orcid = team_member.orcid +LEFT JOIN + person_avatars_by_name() ON person_avatars_by_name.display_name = CONCAT(team_member.given_names,' ', team_member.family_names) +LEFT JOIN + person_avatars_by_orcid() ON person_avatars_by_orcid.orcid = team_member.orcid +$$; + +--ROLES ALREADY IN RSD +--Use this to suggest roles in the modal +CREATE FUNCTION suggested_roles() RETURNS +VARCHAR[] LANGUAGE sql STABLE AS +$$ + SELECT + ARRAY_AGG("role") + FROM ( + SELECT + "role" + FROM + contributor + WHERE + "role" IS NOT NULL + UNION + SELECT + "role" + FROM + team_member + WHERE + "role" IS NOT NULL + ) roles; +$$; diff --git a/database/104-software-views.sql b/database/104-software-views.sql new file mode 100644 index 0000000..f6b1a68 --- /dev/null +++ b/database/104-software-views.sql @@ -0,0 +1,833 @@ +-- SPDX-FileCopyrightText: 2023 - 2025 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2023 - 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2023 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2023 dv4all +-- SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +-- SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE FUNCTION delete_software(id UUID) +RETURNS VOID +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +AS +$$ +BEGIN + IF id IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please provide the ID of the software to delete'; + END IF; + + IF + (SELECT rolsuper FROM pg_roles WHERE rolname = SESSION_USER) IS DISTINCT FROM TRUE + AND + (SELECT CURRENT_SETTING('request.jwt.claims', FALSE)::json->>'role') IS DISTINCT FROM 'rsd_admin' + THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to delete this software'; + END IF; + + DELETE FROM category_for_software WHERE category_for_software.software_id = delete_software.id; + DELETE FROM contributor WHERE contributor.software = delete_software.id; + DELETE FROM invite_maintainer_for_software WHERE invite_maintainer_for_software.software = delete_software.id; + DELETE FROM keyword_for_software WHERE keyword_for_software.software = delete_software.id; + DELETE FROM license_for_software WHERE license_for_software.software = delete_software.id; + DELETE FROM maintainer_for_software WHERE maintainer_for_software.software = delete_software.id; + DELETE FROM mention_for_software WHERE mention_for_software.software = delete_software.id; + DELETE FROM package_manager WHERE package_manager.software = delete_software.id; + DELETE FROM reference_paper_for_software WHERE reference_paper_for_software.software = delete_software.id; + DELETE FROM release_version WHERE release_version.release_id = delete_software.id; + DELETE FROM release WHERE release.software = delete_software.id; + DELETE FROM repository_url WHERE repository_url.software = delete_software.id; + DELETE FROM software_for_community WHERE software_for_community.software = delete_software.id; + DELETE FROM software_for_organisation WHERE software_for_organisation.software = delete_software.id; + DELETE FROM software_for_project WHERE software_for_project.software = delete_software.id; + DELETE FROM software_for_software WHERE software_for_software.origin = delete_software.id OR software_for_software.relation = delete_software.id; + DELETE FROM software_highlight WHERE software_highlight.software = delete_software.id; + DELETE FROM testimonial WHERE testimonial.software = delete_software.id; + + DELETE FROM software WHERE software.id = delete_software.id; +END +$$; + + +CREATE FUNCTION reference_papers_to_scrape() +RETURNS TABLE ( + id UUID, + doi CITEXT, + openalex_id CITEXT, + citations_scraped_at TIMESTAMPTZ, + known_citing_dois CITEXT[] +) +LANGUAGE sql STABLE AS +$$ + SELECT mention.id, mention.doi, mention.openalex_id, mention.citations_scraped_at, ARRAY_REMOVE(ARRAY_AGG(citation.doi), NULL) + FROM mention + LEFT JOIN citation_for_mention ON mention.id = citation_for_mention.mention + LEFT JOIN mention AS citation ON citation_for_mention.citation = citation.id + WHERE + -- ONLY items with DOI or OpenAlex id + (mention.doi IS NOT NULL OR mention.openalex_id IS NOT NULL) + AND ( + mention.id IN ( + SELECT mention FROM reference_paper_for_software + ) + OR + mention.id IN ( + SELECT mention FROM output_for_project + ) + ) + GROUP BY mention.id +$$; + +-- UNIQUE CITATIONS BY SOFTWARE ID +CREATE FUNCTION citation_by_software() RETURNS TABLE ( + software UUID, + id UUID, + doi CITEXT, + url VARCHAR, + title VARCHAR, + authors VARCHAR, + publisher VARCHAR, + publication_year SMALLINT, + journal VARCHAR, + page VARCHAR, + image_url VARCHAR, + mention_type mention_type, + source VARCHAR, + reference_papers UUID[] +) LANGUAGE sql STABLE AS +$$ +SELECT + reference_paper_for_software.software, + mention.id, + mention.doi, + mention.url, + mention.title, + mention.authors, + mention.publisher, + mention.publication_year, + mention.journal, + mention.page, + mention.image_url, + mention.mention_type, + mention.source, + ARRAY_AGG( + reference_paper_for_software.mention + ) AS reference_paper +FROM + reference_paper_for_software +INNER JOIN + citation_for_mention ON citation_for_mention.mention = reference_paper_for_software.mention +INNER JOIN + mention ON mention.id = citation_for_mention.citation +--EXCLUDE reference papers items from citations +WHERE + mention.id NOT IN ( + SELECT mention FROM reference_paper_for_software + ) +GROUP BY + reference_paper_for_software.software, + mention.id +; +$$; + +-- UNIQUE MENTIONS & CITATIONS BY SOFTWARE ID +-- UNION will deduplicate exact entries +CREATE FUNCTION mentions_by_software() RETURNS TABLE ( + software UUID, + id UUID, + doi CITEXT, + url VARCHAR, + title VARCHAR, + authors VARCHAR, + publisher VARCHAR, + publication_year SMALLINT, + journal VARCHAR, + page VARCHAR, + image_url VARCHAR, + mention_type mention_type, + source VARCHAR +) LANGUAGE sql STABLE AS +$$ +WITH mentions_and_citations AS ( + -- mentions for software + SELECT + mention_for_software.software, + mention.id, + mention.doi, + mention.url, + mention.title, + mention.authors, + mention.publisher, + mention.publication_year, + mention.journal, + mention.page, + mention.image_url, + mention.mention_type, + mention.source + FROM + mention + INNER JOIN + mention_for_software ON mention_for_software.mention = mention.id + -- does not deduplicate identical entries, but we will do so below with DISTINCT + -- from scraped citations + UNION ALL + -- scraped citations from reference papers + SELECT + software, + id, + doi, + url, + title, + authors, + publisher, + publication_year, + journal, + page, + image_url, + mention_type, + source + FROM + citation_by_software() +) +SELECT DISTINCT ON (mentions_and_citations.software, mentions_and_citations.id) * FROM mentions_and_citations; +$$; + +-- COUNT mentions per software +CREATE FUNCTION count_software_mentions() RETURNS TABLE (software UUID, mention_cnt BIGINT) LANGUAGE sql STABLE AS +$$ + SELECT + mentions_by_software.software, COUNT(mentions_by_software.id) AS mention_cnt + FROM + mentions_by_software() + GROUP BY + mentions_by_software.software; +$$; + +CREATE MATERIALIZED VIEW count_software_mentions_cached AS SELECT * FROM count_software_mentions(); + +CREATE UNIQUE INDEX ON count_software_mentions_cached(software); + +-- CATEGORIES for software overview filter (GLOBAL) +CREATE FUNCTION software_categories() RETURNS TABLE( + software UUID, + category VARCHAR[] +) LANGUAGE sql STABLE AS +$$ + SELECT + category_for_software.software_id AS software, + ARRAY_AGG( + DISTINCT category_path.short_name + ORDER BY category_path.short_name + ) AS category + FROM + category_for_software + INNER JOIN + category_path(category_for_software.category_id) ON TRUE + WHERE + -- FILTER FOR GLOBAL CATEGORIES + category_path.community IS NULL AND category_path.organisation IS NULL + GROUP BY + category_for_software.software_id; +$$; + +-- SOFTWARE OVERVIEW LIST +-- WITH COUNTS and KEYWORDS for filtering +-- USED BY remote RSD scraper! +CREATE FUNCTION software_overview() RETURNS TABLE ( + id UUID, + slug VARCHAR, + brand_name VARCHAR, + short_statement VARCHAR, + image_id VARCHAR, + updated_at TIMESTAMPTZ, + contributor_cnt BIGINT, + mention_cnt BIGINT, + is_published BOOLEAN, + keywords CITEXT[], + keywords_text TEXT, + prog_lang TEXT[], + licenses VARCHAR[], + categories VARCHAR[] +) LANGUAGE sql STABLE AS +$$ +SELECT + software.id, + software.slug, + software.brand_name, + software.short_statement, + software.image_id, + software.updated_at, + count_software_contributors.contributor_cnt, + count_software_mentions_cached.mention_cnt, + software.is_published, + keyword_filter_for_software.keywords, + keyword_filter_for_software.keywords_text, + prog_lang_filter_for_software.prog_lang, + license_filter_for_software.licenses, + software_categories.category AS categories +FROM + software +LEFT JOIN + count_software_contributors() ON software.id=count_software_contributors.software +LEFT JOIN + count_software_mentions_cached ON software.id=count_software_mentions_cached.software +LEFT JOIN + keyword_filter_for_software() ON software.id=keyword_filter_for_software.software +LEFT JOIN + prog_lang_filter_for_software() ON software.id=prog_lang_filter_for_software.software +LEFT JOIN + license_filter_for_software() ON software.id=license_filter_for_software.software +LEFT JOIN + software_categories() ON software.id=software_categories.software +; +$$; + +-- HIGHLIGHT OVERVIEW LIST +-- WITH COUNTS and KEYWORDS for filtering +CREATE FUNCTION highlight_overview() RETURNS TABLE ( + id UUID, + slug VARCHAR, + brand_name VARCHAR, + short_statement VARCHAR, + image_id VARCHAR, + updated_at TIMESTAMPTZ, + contributor_cnt BIGINT, + mention_cnt BIGINT, + is_published BOOLEAN, + keywords CITEXT[], + keywords_text TEXT, + prog_lang TEXT[], + licenses VARCHAR[], + categories VARCHAR[], + "position" INT +) LANGUAGE sql STABLE AS +$$ +SELECT + software_overview.id, + software_overview.slug, + software_overview.brand_name, + software_overview.short_statement, + software_overview.image_id, + software_overview.updated_at, + software_overview.contributor_cnt, + software_overview.mention_cnt, + software_overview.is_published, + software_overview.keywords, + software_overview.keywords_text, + software_overview.prog_lang, + software_overview.licenses, + software_overview.categories, + software_highlight.position +FROM + software_overview() +RIGHT JOIN + software_highlight ON software_overview.id=software_highlight.software +; +$$; + +-- SOFTWARE OVERVIEW LIST FOR SEARCH +-- WITH COUNTS and KEYWORDS for filtering +CREATE FUNCTION software_search(search VARCHAR) RETURNS TABLE ( + id UUID, + slug VARCHAR, + brand_name VARCHAR, + short_statement VARCHAR, + image_id VARCHAR, + updated_at TIMESTAMPTZ, + is_published BOOLEAN, + contributor_cnt BIGINT, + mention_cnt BIGINT, + keywords CITEXT[], + keywords_text TEXT, + prog_lang TEXT[], + licenses VARCHAR[], + categories VARCHAR[] +) LANGUAGE sql STABLE AS +$$ +SELECT + software.id, + software.slug, + software.brand_name, + software.short_statement, + software.image_id, + software.updated_at, + software.is_published, + count_software_contributors.contributor_cnt, + count_software_mentions.mention_cnt, + keyword_filter_for_software.keywords, + keyword_filter_for_software.keywords_text, + prog_lang_filter_for_software.prog_lang, + license_filter_for_software.licenses, + software_categories.category AS categories +FROM + software +LEFT JOIN + count_software_contributors() ON software.id=count_software_contributors.software +LEFT JOIN + count_software_mentions() ON software.id=count_software_mentions.software +LEFT JOIN + keyword_filter_for_software() ON software.id=keyword_filter_for_software.software +LEFT JOIN + prog_lang_filter_for_software() ON software.id=prog_lang_filter_for_software.software +LEFT JOIN + license_filter_for_software() ON software.id=license_filter_for_software.software +LEFT JOIN + software_categories() ON software.id=software_categories.software +WHERE + software.brand_name ILIKE CONCAT('%', search, '%') + OR + software.slug ILIKE CONCAT('%', search, '%') + OR + software.short_statement ILIKE CONCAT('%', search, '%') + OR + keyword_filter_for_software.keywords_text ILIKE CONCAT('%', search, '%') +ORDER BY + CASE + WHEN brand_name ILIKE search THEN 0 + WHEN brand_name ILIKE CONCAT(search, '%') THEN 1 + WHEN brand_name ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END, + CASE + WHEN slug ILIKE search THEN 0 + WHEN slug ILIKE CONCAT(search, '%') THEN 1 + WHEN slug ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END, + CASE + WHEN short_statement ILIKE search THEN 0 + WHEN short_statement ILIKE CONCAT(search, '%') THEN 1 + WHEN short_statement ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END +; +$$; + +-- HIGHLIGHT OVERVIEW LIST FOR SEARCH +-- WITH COUNTS and KEYWORDS for filtering +CREATE FUNCTION highlight_search(search VARCHAR) RETURNS TABLE ( + id UUID, + slug VARCHAR, + brand_name VARCHAR, + short_statement VARCHAR, + image_id VARCHAR, + updated_at TIMESTAMPTZ, + is_published BOOLEAN, + contributor_cnt BIGINT, + mention_cnt BIGINT, + keywords CITEXT[], + keywords_text TEXT, + prog_lang TEXT[], + licenses VARCHAR[], + categories VARCHAR[], + "position" INT +) LANGUAGE sql STABLE AS +$$ +SELECT + software_search.id, + software_search.slug, + software_search.brand_name, + software_search.short_statement, + software_search.image_id, + software_search.updated_at, + software_search.is_published, + software_search.contributor_cnt, + software_search.mention_cnt, + software_search.keywords, + software_search.keywords_text, + software_search.prog_lang, + software_search.licenses, + software_search.categories, + software_highlight.position +FROM + software_search(search) +INNER JOIN + software_highlight ON software_search.id=software_highlight.software +; +$$; + +-- Get a list of all software highlights +CREATE FUNCTION software_for_highlight() RETURNS TABLE ( + id UUID, + slug VARCHAR, + brand_name VARCHAR, + short_statement VARCHAR, + image_id VARCHAR, + updated_at TIMESTAMPTZ, + is_published BOOLEAN, + contributor_cnt BIGINT, + mention_cnt BIGINT, + keywords CITEXT[], + keywords_text TEXT, + prog_lang TEXT[], + licenses VARCHAR[], + "position" INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + software.id, + software.slug, + software.brand_name, + software.short_statement, + software.image_id, + software.updated_at, + software.is_published, + count_software_contributors.contributor_cnt, + count_software_mentions_cached.mention_cnt, + keyword_filter_for_software.keywords, + keyword_filter_for_software.keywords_text, + prog_lang_filter_for_software.prog_lang, + license_filter_for_software.licenses, + software_highlight.position +FROM + software +INNER JOIN + software_highlight ON software.id=software_highlight.software +LEFT JOIN + count_software_contributors() ON software.id=count_software_contributors.software +LEFT JOIN + count_software_mentions_cached ON software.id=count_software_mentions_cached.software +LEFT JOIN + keyword_filter_for_software() ON software.id=keyword_filter_for_software.software +LEFT JOIN + prog_lang_filter_for_software() ON software.id=prog_lang_filter_for_software.software +LEFT JOIN + license_filter_for_software() ON software.id=license_filter_for_software.software +; +$$; + +-- REACTIVE KEYWORD FILTER WITH COUNTS FOR HIGHLIGHTS +-- PROVIDES AVAILABLE KEYWORDS FOR APPLIED FILTERS +CREATE FUNCTION highlight_keywords_filter( + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + keyword CITEXT, + keyword_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(keywords) AS keyword, + COUNT(id) AS keyword_cnt +FROM + highlight_search(search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + COALESCE(categories, '{}') @> category_filter +GROUP BY + keyword +; +$$; + +-- REACTIVE PROGRAMMING LANGUAGES WITH COUNTS FOR HIGHLIGHTS +-- PROVIDES AVAILABLE PROGRAMMING LANGUAGES FOR APPLIED FILTERS +CREATE FUNCTION highlight_languages_filter( + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + prog_language TEXT, + prog_language_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(prog_lang) AS prog_language, + COUNT(id) AS prog_language_cnt +FROM + highlight_search(search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + COALESCE(categories, '{}') @> category_filter +GROUP BY + prog_language +; +$$; + +-- REACTIVE LICENSES FILTER WITH COUNTS FOR HIGHLIGHTS +-- PROVIDES AVAILABLE LICENSES FOR APPLIED FILTERS +CREATE FUNCTION highlight_licenses_filter( + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + license VARCHAR, + license_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(licenses) AS license, + COUNT(id) AS license_cnt +FROM + highlight_search(search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + COALESCE(categories, '{}') @> category_filter +GROUP BY + license +; +$$; + +-- REACTIVE CATEGORIES FILTER WITH COUNTS FOR HIGHLIGHTS +-- PROVIDES AVAILABLE CATEGORIES FOR APPLIED FILTERS +CREATE FUNCTION highlight_category_filter( + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + category VARCHAR, + category_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(categories) AS category, + COUNT(DISTINCT(id)) AS category_cnt +FROM + highlight_search(search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + COALESCE(categories, '{}') @> category_filter +GROUP BY + category +; +$$; + +-- FILTER SOFTWARE by ORCID or account ID +-- OPT-IN ONLY, uses public_user_profile as filter +-- UNIQUE entries by software.id +CREATE FUNCTION software_by_public_profile() RETURNS TABLE ( + id UUID, + slug VARCHAR, + brand_name VARCHAR, + short_statement VARCHAR, + image_id VARCHAR, + updated_at TIMESTAMPTZ, + contributor_cnt BIGINT, + mention_cnt BIGINT, + is_published BOOLEAN, + keywords CITEXT[], + keywords_text TEXT, + prog_lang TEXT[], + licenses VARCHAR[], + orcid VARCHAR, + account UUID +) LANGUAGE sql STABLE AS +$$ +SELECT + software.id, + software.slug, + software.brand_name, + software.short_statement, + software.image_id, + software.updated_at, + count_software_contributors.contributor_cnt, + count_software_mentions.mention_cnt, + software.is_published, + keyword_filter_for_software.keywords, + keyword_filter_for_software.keywords_text, + prog_lang_filter_for_software.prog_lang, + license_filter_for_software.licenses, + public_user_profile.orcid, + public_user_profile.account +FROM + public_user_profile() +INNER JOIN + contributor ON ( + public_user_profile.orcid = contributor.orcid + OR public_user_profile.account = contributor.account + ) +LEFT JOIN + software ON software.id = contributor.software +LEFT JOIN + count_software_contributors() ON software.id=count_software_contributors.software +LEFT JOIN + count_software_mentions() ON software.id=count_software_mentions.software +LEFT JOIN + keyword_filter_for_software() ON software.id=keyword_filter_for_software.software +LEFT JOIN + prog_lang_filter_for_software() ON software.id=prog_lang_filter_for_software.software +LEFT JOIN + license_filter_for_software() ON software.id=license_filter_for_software.software +; +$$; +-- contributors for software +-- public_profile.orcid indicates public profile OPT-IN +CREATE FUNCTION software_contributors(software_id UUID) RETURNS TABLE ( + id UUID, + is_contact_person BOOLEAN, + email_address VARCHAR, + family_names VARCHAR, + given_names VARCHAR, + affiliation VARCHAR, + role VARCHAR, + orcid VARCHAR, + avatar_id VARCHAR, + "position" INT, + software UUID, + account UUID, + is_public VARCHAR +) LANGUAGE sql STABLE AS +$$ +SELECT DISTINCT ON (contributor.id) + contributor.id, + contributor.is_contact_person, + contributor.email_address, + contributor.family_names, + contributor.given_names, + contributor.affiliation, + contributor.role, + contributor.orcid, + contributor.avatar_id, + contributor."position", + contributor.software, + public_user_profile.account, + public_user_profile.is_public +FROM + contributor +LEFT JOIN + public_user_profile() ON ( + contributor.orcid = public_user_profile.orcid + OR + contributor.account = public_user_profile.account + ) +WHERE + contributor.software = software_id +; +$$; + +-- RELATED SOFTWARE LIST WITH COUNTS +CREATE FUNCTION related_software_for_software(software_id UUID) RETURNS TABLE ( + id UUID, + slug VARCHAR, + brand_name VARCHAR, + short_statement VARCHAR, + updated_at TIMESTAMPTZ, + contributor_cnt BIGINT, + mention_cnt BIGINT, + is_published BOOLEAN +) LANGUAGE sql STABLE AS +$$ + SELECT + software.id, + software.slug, + software.brand_name, + software.short_statement, + software.updated_at, + count_software_contributors.contributor_cnt, + count_software_mentions.mention_cnt, + software.is_published + FROM + software + LEFT JOIN + count_software_contributors() ON software.id=count_software_contributors.software + LEFT JOIN + count_software_mentions() ON software.id=count_software_mentions.software + INNER JOIN + software_for_software ON software.id=software_for_software.relation + WHERE + software_for_software.origin = software_id; +$$; + +-- RELATED SOFTWARE for PROJECT +-- filter by project_id +CREATE FUNCTION related_software_for_project(project_id UUID) RETURNS TABLE ( + id UUID, + slug VARCHAR, + brand_name VARCHAR, + short_statement VARCHAR, + updated_at TIMESTAMPTZ, + contributor_cnt BIGINT, + mention_cnt BIGINT, + is_published BOOLEAN, + status relation_status +) LANGUAGE sql STABLE AS +$$ + SELECT + software.id, + software.slug, + software.brand_name, + software.short_statement, + software.updated_at, + count_software_contributors.contributor_cnt, + count_software_mentions.mention_cnt, + software.is_published, + software_for_project.status + FROM + software + LEFT JOIN + count_software_contributors() ON software.id=count_software_contributors.software + LEFT JOIN + count_software_mentions() ON software.id=count_software_mentions.software + INNER JOIN + software_for_project ON software.id=software_for_project.software + WHERE + software_for_project.project=project_id; +$$; + +-- SOFTWARE BY MAINTAINER +-- NOTE! one software is shown multiple times in this view +-- we filter this view at least by organisation uuid +CREATE FUNCTION software_by_maintainer(maintainer_id UUID) RETURNS TABLE ( + id UUID, + slug VARCHAR, + brand_name VARCHAR, + short_statement VARCHAR, + is_published BOOLEAN, + image_id VARCHAR, + updated_at TIMESTAMPTZ, + contributor_cnt BIGINT, + mention_cnt BIGINT +) LANGUAGE sql STABLE AS +$$ + SELECT + software.id, + software.slug, + software.brand_name, + software.short_statement, + software.is_published, + software.image_id, + software.updated_at, + count_software_contributors.contributor_cnt, + count_software_mentions.mention_cnt + FROM + software + LEFT JOIN + count_software_contributors() ON software.id=count_software_contributors.software + LEFT JOIN + count_software_mentions() ON software.id=count_software_mentions.software + INNER JOIN + maintainer_for_software ON software.id=maintainer_for_software.software + WHERE + maintainer_for_software.maintainer=maintainer_id; +$$; diff --git a/database/106-project-views.sql b/database/106-project-views.sql new file mode 100644 index 0000000..b1680d2 --- /dev/null +++ b/database/106-project-views.sql @@ -0,0 +1,940 @@ +-- SPDX-FileCopyrightText: 2023 - 2025 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2023 - 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2023 Dusan Mijatovic (dv4all) +-- SPDX-FileCopyrightText: 2023 dv4all +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE FUNCTION delete_project(id UUID) +RETURNS VOID +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +AS +$$ +BEGIN + IF id IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please provide the ID of the project to delete'; + END IF; + + IF + (SELECT rolsuper FROM pg_roles WHERE rolname = SESSION_USER) IS DISTINCT FROM TRUE + AND + (SELECT CURRENT_SETTING('request.jwt.claims', FALSE)::json->>'role') IS DISTINCT FROM 'rsd_admin' + THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to delete this project'; + END IF; + + DELETE FROM category_for_project WHERE category_for_project.project_id = delete_project.id; + DELETE FROM impact_for_project WHERE impact_for_project.project = delete_project.id; + DELETE FROM invite_maintainer_for_project WHERE invite_maintainer_for_project.project = delete_project.id; + DELETE FROM keyword_for_project WHERE keyword_for_project.project = delete_project.id; + DELETE FROM maintainer_for_project WHERE maintainer_for_project.project = delete_project.id; + DELETE FROM output_for_project WHERE output_for_project.project = delete_project.id; + DELETE FROM project_for_organisation WHERE project_for_organisation.project = delete_project.id; + DELETE FROM project_for_project WHERE project_for_project.origin = delete_project.id OR project_for_project.relation = delete_project.id; + DELETE FROM research_domain_for_project WHERE research_domain_for_project.project = delete_project.id; + DELETE FROM software_for_project WHERE software_for_project.project = delete_project.id; + DELETE FROM team_member WHERE team_member.project = delete_project.id; + DELETE FROM testimonial_for_project WHERE testimonial_for_project.project = delete_project.id; + DELETE FROM url_for_project WHERE url_for_project.project = delete_project.id; + + DELETE FROM project WHERE project.id = delete_project.id; +END +$$; + +-- UNIQUE CITATIONS BY PROJECT ID +-- Scraped citation using output as reference +CREATE FUNCTION citation_by_project() RETURNS TABLE ( + project UUID, + id UUID, + doi CITEXT, + url VARCHAR, + title VARCHAR, + authors VARCHAR, + publisher VARCHAR, + publication_year SMALLINT, + journal VARCHAR, + page VARCHAR, + image_url VARCHAR, + mention_type mention_type, + source VARCHAR, + reference_papers UUID[] +) LANGUAGE sql STABLE AS +$$ +SELECT + output_for_project.project, + mention.id, + mention.doi, + mention.url, + mention.title, + mention.authors, + mention.publisher, + mention.publication_year, + mention.journal, + mention.page, + mention.image_url, + mention.mention_type, + mention.source, + ARRAY_AGG( + output_for_project.mention + ) AS reference_paper +FROM + output_for_project +INNER JOIN + citation_for_mention ON citation_for_mention.mention = output_for_project.mention +INNER JOIN + mention ON mention.id = citation_for_mention.citation +--EXCLUDE reference papers items from citations +WHERE + mention.id NOT IN ( + SELECT mention FROM output_for_project + ) +GROUP BY + output_for_project.project, + mention.id +; +$$; + + +-- UNIQUE MENTIONS & CITATIONS BY PROJECT ID +-- UNION will deduplicate exact entries +CREATE FUNCTION impact_by_project() RETURNS TABLE ( + project UUID, + id UUID, + doi CITEXT, + url VARCHAR, + title VARCHAR, + authors VARCHAR, + publisher VARCHAR, + publication_year SMALLINT, + journal VARCHAR, + page VARCHAR, + image_url VARCHAR, + mention_type mention_type, + source VARCHAR, + note VARCHAR +) LANGUAGE sql STABLE AS +$$ +WITH impact_and_citations AS ( + -- impact for project + SELECT + impact_for_project.project, + mention.id, + mention.doi, + mention.url, + mention.title, + mention.authors, + mention.publisher, + mention.publication_year, + mention.journal, + mention.page, + mention.image_url, + mention.mention_type, + mention.source, + mention.note + FROM + mention + INNER JOIN + impact_for_project ON impact_for_project.mention = mention.id + -- does not deduplicate identical entries, but we will do so below with DISTINCT + -- from scraped citations + UNION ALL + -- scraped citations from reference papers + SELECT + project, + id, + doi, + url, + title, + authors, + publisher, + publication_year, + journal, + page, + image_url, + mention_type, + source, + -- scraped citations have no note prop + -- we need this prop in the edit impact section + NULL as note + FROM + citation_by_project() +) SELECT DISTINCT ON (impact_and_citations.project, impact_and_citations.id) * FROM impact_and_citations; +; +$$; + + +CREATE FUNCTION count_project_team_members() RETURNS TABLE ( + project UUID, + team_member_cnt INTEGER, + has_contact_person BOOLEAN +) LANGUAGE sql STABLE AS +$$ +SELECT + team_member.project, + COUNT(team_member.id), + BOOL_OR(team_member.is_contact_person) +FROM + team_member +GROUP BY + team_member.project; +$$; + + +CREATE FUNCTION count_project_related_software() RETURNS TABLE ( + project UUID, + software_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + software_for_project.project, + COUNT(software_for_project.software) +FROM + software_for_project +WHERE + software_for_project.status = 'approved' +GROUP BY + software_for_project.project; +$$; + + +CREATE FUNCTION count_project_related_projects() RETURNS TABLE ( + project UUID, + project_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + id, + COUNT(DISTINCT relations) +FROM +( + SELECT + project_for_project.origin AS id, + UNNEST(ARRAY_AGG(project_for_project.relation)) AS relations + FROM + project_for_project + WHERE + project_for_project.status = 'approved' + GROUP BY + project_for_project.origin + UNION ALL + SELECT + project_for_project.relation AS id, + UNNEST(ARRAY_AGG(project_for_project.origin)) AS relations + FROM + project_for_project + WHERE + project_for_project.status = 'approved' + GROUP BY + project_for_project.relation +) AS cnts +GROUP BY id; +$$; + + +CREATE FUNCTION count_project_organisations() RETURNS TABLE ( + project UUID, + participating_org_cnt INTEGER, + funding_org_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + project_for_organisation.project, + COUNT(CASE WHEN project_for_organisation.role = 'participating' THEN 1 END), + COUNT(CASE WHEN project_for_organisation.role = 'funding' THEN 1 END) +FROM + project_for_organisation +WHERE + project_for_organisation.status = 'approved' +GROUP BY + project_for_organisation.project; +$$; + + +CREATE FUNCTION count_project_keywords() RETURNS TABLE ( + project UUID, + keyword_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + keyword_for_project.project, + COUNT(keyword_for_project.keyword) +FROM + keyword_for_project +GROUP BY + keyword_for_project.project; +$$; + + +CREATE FUNCTION count_project_research_domains() RETURNS TABLE ( + project UUID, + research_domain_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + research_domain_for_project.project, + COUNT(research_domain_for_project.research_domain) +FROM + research_domain_for_project +GROUP BY + research_domain_for_project.project; +$$; + + +CREATE FUNCTION count_project_impact() RETURNS TABLE ( + project UUID, + impact_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + impact_by_project.project, + COUNT(impact_by_project.id) +FROM + impact_by_project() +GROUP BY + impact_by_project.project; +$$; + + +CREATE FUNCTION count_project_output() RETURNS TABLE ( + project UUID, + output_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + output_for_project.project, + COUNT(output_for_project.mention) +FROM + output_for_project +GROUP BY + output_for_project.project; +$$; + +CREATE FUNCTION project_status() RETURNS TABLE ( + project UUID, + status VARCHAR(20) +) LANGUAGE sql STABLE AS +$$ +SELECT + project.id, + CASE + WHEN project.date_end < now() THEN 'finished'::VARCHAR + WHEN project.date_start > now() THEN 'upcoming'::VARCHAR + WHEN project.date_start < now() AND project.date_end > now() THEN 'in_progress'::VARCHAR + ELSE 'unknown'::VARCHAR + END AS status +FROM + project +$$; + +CREATE FUNCTION project_quality(show_all BOOLEAN DEFAULT FALSE) RETURNS TABLE ( + slug VARCHAR, + title VARCHAR, + has_subtitle BOOLEAN, + is_published BOOLEAN, + date_start DATE, + date_end DATE, + grant_id VARCHAR, + has_image BOOLEAN, + team_member_cnt INTEGER, + has_contact_person BOOLEAN, + participating_org_cnt INTEGER, + funding_org_cnt INTEGER, + software_cnt INTEGER, + project_cnt INTEGER, + keyword_cnt INTEGER, + research_domain_cnt INTEGER, + impact_cnt INTEGER, + output_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + project.slug, + project.title, + project.subtitle IS NOT NULL, + project.is_published, + project.date_start, + project.date_end, + project.grant_id, + project.image_id IS NOT NULL, + COALESCE(count_project_team_members.team_member_cnt, 0), + COALESCE(count_project_team_members.has_contact_person, FALSE), + COALESCE(count_project_organisations.participating_org_cnt, 0), + COALESCE(count_project_organisations.funding_org_cnt, 0), + COALESCE(count_project_related_software.software_cnt, 0), + COALESCE(count_project_related_projects.project_cnt, 0), + COALESCE(count_project_keywords.keyword_cnt, 0), + COALESCE(count_project_research_domains.research_domain_cnt, 0), + COALESCE(count_project_impact.impact_cnt, 0), + COALESCE(count_project_output.output_cnt, 0) +FROM + project +LEFT JOIN + count_project_team_members() ON project.id = count_project_team_members.project +LEFT JOIN + count_project_organisations() ON project.id = count_project_organisations.project +LEFT JOIN + count_project_related_software() ON project.id = count_project_related_software.project +LEFT JOIN + count_project_related_projects() ON project.id = count_project_related_projects.project +LEFT JOIN + count_project_keywords() ON project.id = count_project_keywords.project +LEFT JOIN + count_project_research_domains() ON project.id = count_project_research_domains.project +LEFT JOIN + count_project_impact() ON project.id = count_project_impact.project +LEFT JOIN + count_project_output() ON project.id = count_project_output.project +WHERE + CASE WHEN show_all IS TRUE THEN TRUE ELSE project.id IN (SELECT * FROM projects_of_current_maintainer()) END; +$$; + +-- RELATED PROJECTS for project_id +-- use bi-directional linking: project_id used as origin or relation +CREATE FUNCTION related_projects_for_project(project_id UUID) RETURNS TABLE ( + id UUID, + slug VARCHAR, + title VARCHAR, + subtitle VARCHAR, + current_state VARCHAR, + date_start DATE, + updated_at TIMESTAMPTZ, + is_published BOOLEAN, + image_contain BOOLEAN, + image_id VARCHAR, + status relation_status, + origin UUID, + relation UUID +) LANGUAGE sql STABLE AS +$$ +SELECT DISTINCT ON (project.id) + project.id, + project.slug, + project.title, + project.subtitle, + project_status.status AS current_state, + project.date_start, + project.updated_at, + project.is_published, + project.image_contain, + project.image_id, + project_for_project.status, + project_for_project.origin, + project_for_project.relation +FROM + project +INNER JOIN + project_for_project ON + (project.id = project_for_project.relation AND project_for_project.origin = project_id) + OR + (project.id = project_for_project.origin AND project_for_project.relation = project_id) +LEFT JOIN + project_status() ON project.id=project_status.project +; +$$; + +-- AGGREGATE participating organisations per project for project_overview RPC +-- use only TOP LEVEL organisations (parent IS NULL) +CREATE FUNCTION project_participating_organisations() RETURNS TABLE ( + project UUID, + organisations VARCHAR[] +) LANGUAGE sql STABLE AS +$$ +SELECT + project_for_organisation.project, + ARRAY_AGG(organisation.name) AS organisations +FROM + organisation +INNER JOIN + project_for_organisation ON organisation.id = project_for_organisation.organisation +WHERE + project_for_organisation.role = 'participating' AND organisation.parent IS NULL +GROUP BY + project_for_organisation.project +; +$$; + + +-- PROJECT OVERVIEW LIST +-- WHEN FILTERING/SEARCH IS NOT USED +CREATE FUNCTION project_overview() RETURNS TABLE ( + id UUID, + slug VARCHAR, + title VARCHAR, + subtitle VARCHAR, + date_start DATE, + date_end DATE, + updated_at TIMESTAMPTZ, + is_published BOOLEAN, + image_contain BOOLEAN, + image_id VARCHAR, + keywords citext[], + keywords_text TEXT, + research_domain VARCHAR[], + research_domain_text TEXT, + participating_organisations VARCHAR[], + impact_cnt INTEGER, + output_cnt INTEGER, + project_status VARCHAR(20) +) LANGUAGE sql STABLE AS +$$ +SELECT + project.id, + project.slug, + project.title, + project.subtitle, + project.date_start, + project.date_end, + project.updated_at, + project.is_published, + project.image_contain, + project.image_id, + keyword_filter_for_project.keywords, + keyword_filter_for_project.keywords_text, + research_domain_filter_for_project.research_domain, + research_domain_filter_for_project.research_domain_text, + project_participating_organisations.organisations AS participating_organisations, + COALESCE(count_project_impact.impact_cnt, 0) AS impact_cnt, + COALESCE(count_project_output.output_cnt, 0) AS output_cnt, + project_status.status +FROM + project +LEFT JOIN + keyword_filter_for_project() ON project.id=keyword_filter_for_project.project +LEFT JOIN + research_domain_filter_for_project() ON project.id=research_domain_filter_for_project.project +LEFT JOIN + project_participating_organisations() ON project.id=project_participating_organisations.project +LEFT JOIN + count_project_impact() ON project.id=count_project_impact.project +LEFT JOIN + count_project_output() ON project.id=count_project_output.project +LEFT JOIN + project_status() ON project.id=project_status.project +; +$$; + +-- PROJECT OVERVIEW LIST FOR SEARCH +-- WITH keywords, research domain and participating organisations for filtering +CREATE FUNCTION project_search(search VARCHAR) RETURNS TABLE ( + id UUID, + slug VARCHAR, + title VARCHAR, + subtitle VARCHAR, + date_start DATE, + date_end DATE, + updated_at TIMESTAMPTZ, + is_published BOOLEAN, + image_contain BOOLEAN, + image_id VARCHAR, + keywords citext[], + keywords_text TEXT, + research_domain VARCHAR[], + research_domain_text TEXT, + participating_organisations VARCHAR[], + impact_cnt INTEGER, + output_cnt INTEGER, + project_status VARCHAR(20) +) LANGUAGE sql STABLE AS +$$ +SELECT + project.id, + project.slug, + project.title, + project.subtitle, + project.date_start, + project.date_end, + project.updated_at, + project.is_published, + project.image_contain, + project.image_id, + keyword_filter_for_project.keywords, + keyword_filter_for_project.keywords_text, + research_domain_filter_for_project.research_domain, + research_domain_filter_for_project.research_domain_text, + project_participating_organisations.organisations AS participating_organisations, + COALESCE(count_project_impact.impact_cnt, 0), + COALESCE(count_project_output.output_cnt, 0), + project_status.status +FROM + project +LEFT JOIN + keyword_filter_for_project() ON project.id=keyword_filter_for_project.project +LEFT JOIN + research_domain_filter_for_project() ON project.id=research_domain_filter_for_project.project +LEFT JOIN + project_participating_organisations() ON project.id=project_participating_organisations.project +LEFT JOIN + count_project_impact() ON project.id = count_project_impact.project +LEFT JOIN + count_project_output() ON project.id = count_project_output.project +LEFT JOIN + project_status() ON project.id=project_status.project +WHERE + project.title ILIKE CONCAT('%', search, '%') + OR + project.slug ILIKE CONCAT('%', search, '%') + OR + project.subtitle ILIKE CONCAT('%', search, '%') + OR + keyword_filter_for_project.keywords_text ILIKE CONCAT('%', search, '%') + OR + research_domain_filter_for_project.research_domain_text ILIKE CONCAT('%', search, '%') +ORDER BY + CASE + WHEN title ILIKE search THEN 0 + WHEN title ILIKE CONCAT(search, '%') THEN 1 + WHEN title ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END, + CASE + WHEN slug ILIKE search THEN 0 + WHEN slug ILIKE CONCAT(search, '%') THEN 1 + WHEN slug ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END, + CASE + WHEN subtitle ILIKE search THEN 0 + WHEN subtitle ILIKE CONCAT(search, '%') THEN 1 + WHEN subtitle ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END +; +$$; + +-- REACTIVE KEYWORD FILTER WITH COUNTS FOR PROJECTS +-- PROVIDES AVAILABLE KEYWORDS FOR APPLIED FILTERS +CREATE FUNCTION project_keywords_filter( + search_filter TEXT DEFAULT '', + status_filter VARCHAR DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + research_domain_filter VARCHAR[] DEFAULT '{}', + organisation_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + keyword CITEXT, + keyword_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(keywords) AS keyword, + COUNT(id) AS keyword_cnt +FROM + project_search(search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(research_domain, '{}') @> research_domain_filter + AND + COALESCE(participating_organisations, '{}') @> organisation_filter + AND + CASE + WHEN status_filter <> '' THEN project_status = status_filter + ELSE true + END +GROUP BY + keyword +; +$$; + + +-- REACTIVE RESEARCH DOMAIN FILTER WITH COUNTS FOR PROJECTS +-- PROVIDES AVAILABLE DOMAINS FOR APPLIED FILTERS +CREATE FUNCTION project_domains_filter( + search_filter TEXT DEFAULT '', + status_filter VARCHAR DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + research_domain_filter VARCHAR[] DEFAULT '{}', + organisation_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + domain VARCHAR, + domain_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(research_domain) AS domain, + COUNT(id) AS domain_cnt +FROM + project_search(search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(research_domain, '{}') @> research_domain_filter + AND + COALESCE(participating_organisations, '{}') @> organisation_filter + AND + CASE + WHEN status_filter <> '' THEN project_status = status_filter + ELSE true + END +GROUP BY + domain +; +$$; + +-- REACTIVE PARTICIPATING ORGANISATIONS FILTER WITH COUNTS FOR PROJECTS +-- PROVIDES AVAILABLE DOMAINS FOR APPLIED FILTERS +CREATE FUNCTION project_participating_organisations_filter( + search_filter TEXT DEFAULT '', + status_filter VARCHAR DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + research_domain_filter VARCHAR[] DEFAULT '{}', + organisation_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + organisation VARCHAR, + organisation_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(participating_organisations) AS organisation, + COUNT(id) AS organisation_cnt +FROM + project_search(search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(research_domain, '{}') @> research_domain_filter + AND + COALESCE(participating_organisations, '{}') @> organisation_filter + AND + CASE + WHEN status_filter <> '' THEN project_status = status_filter + ELSE true + END +GROUP BY + organisation +; +$$; + +-- REACTIVE PROJECT STATUS WITH COUNTS +-- PROVIDES AVAILABLE DOMAINS FOR APPLIED FILTERS +CREATE FUNCTION project_status_filter( + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + research_domain_filter VARCHAR[] DEFAULT '{}', + organisation_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + project_status VARCHAR, + project_status_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + project_status, + COUNT(id) AS project_status_cnt +FROM + project_search(search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(research_domain, '{}') @> research_domain_filter + AND + COALESCE(participating_organisations, '{}') @> organisation_filter +GROUP BY + project_status +; +$$; + +-- FILTER PROJECTS by orcid +-- OPT-IN ONLY, uses public_profile table as filter +-- UNIQUE entries project.id +CREATE FUNCTION project_by_public_profile() RETURNS TABLE ( + id UUID, + slug VARCHAR, + title VARCHAR, + subtitle VARCHAR, + date_start DATE, + date_end DATE, + updated_at TIMESTAMPTZ, + is_published BOOLEAN, + image_contain BOOLEAN, + image_id VARCHAR, + keywords citext[], + keywords_text TEXT, + research_domain VARCHAR[], + participating_organisations VARCHAR[], + impact_cnt INTEGER, + output_cnt INTEGER, + project_status VARCHAR(20), + orcid VARCHAR, + account UUID +) LANGUAGE sql STABLE AS +$$ +SELECT + project.id, + project.slug, + project.title, + project.subtitle, + project.date_start, + project.date_end, + project.updated_at, + project.is_published, + project.image_contain, + project.image_id, + keyword_filter_for_project.keywords, + keyword_filter_for_project.keywords_text, + research_domain_filter_for_project.research_domain, + project_participating_organisations.organisations AS participating_organisations, + COALESCE(count_project_impact.impact_cnt, 0) AS impact_cnt, + COALESCE(count_project_output.output_cnt, 0) AS output_cnt, + project_status.status, + public_user_profile.orcid, + public_user_profile.account +FROM + public_user_profile() +INNER JOIN + team_member ON ( + public_user_profile.orcid = team_member.orcid + OR + public_user_profile.account = team_member.account + ) +LEFT JOIN + project ON project.id=team_member.project +LEFT JOIN + keyword_filter_for_project() ON project.id=keyword_filter_for_project.project +LEFT JOIN + research_domain_filter_for_project() ON project.id=research_domain_filter_for_project.project +LEFT JOIN + project_participating_organisations() ON project.id=project_participating_organisations.project +LEFT JOIN + count_project_impact() ON project.id=count_project_impact.project +LEFT JOIN + count_project_output() ON project.id=count_project_output.project +LEFT JOIN + project_status() ON project.id=project_status.project +; +$$; + +-- FILTER team members for a project by project_id +-- public_orcid_profile indicates public profile OPT-IN +CREATE FUNCTION project_team(project_id UUID) RETURNS TABLE ( + id UUID, + is_contact_person BOOLEAN, + email_address VARCHAR, + family_names VARCHAR, + given_names VARCHAR, + affiliation VARCHAR, + role VARCHAR, + orcid VARCHAR, + avatar_id VARCHAR, + "position" INT, + project UUID, + account UUID, + is_public VARCHAR +) LANGUAGE sql STABLE AS +$$ +SELECT DISTINCT ON (team_member.id) + team_member.id, + team_member.is_contact_person, + team_member.email_address, + team_member.family_names, + team_member.given_names, + team_member.affiliation, + team_member.role, + team_member.orcid, + team_member.avatar_id, + team_member."position", + team_member.project, + public_user_profile.account, + public_user_profile.is_public +FROM + team_member +LEFT JOIN + public_user_profile() ON ( + team_member.orcid = public_user_profile.orcid + OR + team_member.account = public_user_profile.account + ) +WHERE + team_member.project = project_id +; +$$; + + +-- RELATED PROJECTS for software +-- NOTE! updated by Dusan 2022-07-27 +-- filter by software_id +CREATE FUNCTION related_projects_for_software(software_id UUID) RETURNS TABLE ( + software UUID, + id UUID, + slug VARCHAR, + title VARCHAR, + subtitle VARCHAR, + current_state VARCHAR, + date_start DATE, + updated_at TIMESTAMPTZ, + is_published BOOLEAN, + image_contain BOOLEAN, + status relation_status, + image_id VARCHAR +) LANGUAGE sql STABLE AS +$$ + SELECT + software_for_project.software, + project.id, + project.slug, + project.title, + project.subtitle, + project_status.status AS current_state, + project.date_start, + project.updated_at, + project.is_published, + project.image_contain, + software_for_project.status, + project.image_id + FROM + project + INNER JOIN + software_for_project ON project.id = software_for_project.project + LEFT JOIN + project_status() ON project.id=project_status.project + WHERE + software_for_project.software = software_id; +$$; + +-- PROJECTS BY MAINTAINER +-- NOTE! updated by Dusan on 2025-04-07 +-- we filter this view at least by user account (uuid) +CREATE FUNCTION projects_by_maintainer(maintainer_id UUID) RETURNS TABLE ( + id UUID, + slug VARCHAR, + title VARCHAR, + subtitle VARCHAR, + current_state VARCHAR, + date_start DATE, + date_end DATE, + updated_at TIMESTAMPTZ, + is_published BOOLEAN, + image_contain BOOLEAN, + image_id VARCHAR, + impact_cnt INTEGER, + output_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ + SELECT + project.id, + project.slug, + project.title, + project.subtitle, + project_status.status AS current_state, + project.date_start, + project.date_end, + project.updated_at, + project.is_published, + project.image_contain, + project.image_id, + COALESCE(count_project_impact.impact_cnt, 0), + COALESCE(count_project_output.output_cnt, 0) + FROM + project + INNER JOIN + maintainer_for_project ON project.id = maintainer_for_project.project + LEFT JOIN + count_project_impact() ON count_project_impact.project = project.id + LEFT JOIN + count_project_output() ON count_project_output.project = project.id + LEFT JOIN + project_status() ON project.id=project_status.project + WHERE + maintainer_for_project.maintainer = maintainer_id; +$$; diff --git a/database/108-community-views.sql b/database/108-community-views.sql new file mode 100644 index 0000000..e75647d --- /dev/null +++ b/database/108-community-views.sql @@ -0,0 +1,613 @@ +-- SPDX-FileCopyrightText: 2024 - 2025 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE FUNCTION delete_community(id UUID) +RETURNS VOID +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +AS +$$ +BEGIN + IF id IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please provide the ID of the community to delete'; + END IF; + + IF + (SELECT rolsuper FROM pg_roles WHERE rolname = SESSION_USER) IS DISTINCT FROM TRUE + AND + (SELECT CURRENT_SETTING('request.jwt.claims', FALSE)::json->>'role') IS DISTINCT FROM 'rsd_admin' + THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to delete this community'; + END IF; + + DELETE FROM category_for_software WHERE category_id IN (SELECT category.id FROM category WHERE category.community = delete_community.id); + DELETE FROM category WHERE category.community = delete_community.id; + DELETE FROM invite_maintainer_for_community WHERE invite_maintainer_for_community.community = delete_community.id; + DELETE FROM keyword_for_community WHERE keyword_for_community.community = delete_community.id; + DELETE FROM maintainer_for_community WHERE maintainer_for_community.community = delete_community.id; + DELETE FROM software_for_community WHERE software_for_community.community = delete_community.id; + + DELETE FROM community WHERE community.id = delete_community.id; +END +$$; + +-- Software count by community +-- BY DEFAULT we return count of approved software +-- IF public is FALSE we return approved software that is not published too +CREATE FUNCTION software_count_by_community(public BOOLEAN DEFAULT TRUE) RETURNS TABLE ( + community UUID, + software_cnt BIGINT +) LANGUAGE sql STABLE AS +$$ +SELECT + software_for_community.community, + COUNT(DISTINCT software_for_community.software) AS software_cnt +FROM + software_for_community +WHERE + software_for_community.status = 'approved' AND ( + NOT public OR software IN (SELECT id FROM software WHERE is_published) + ) +GROUP BY + software_for_community.community +; +$$; + +-- Pending software count by community +-- BY DEFAULT we return count of approved software +-- IF public is FALSE we return total count (as far as RLS allows) +CREATE FUNCTION pending_count_by_community() RETURNS TABLE ( + community UUID, + pending_cnt BIGINT +) LANGUAGE sql STABLE AS +$$ +SELECT + software_for_community.community, + COUNT(DISTINCT software_for_community.software) AS pending_cnt +FROM + software_for_community +WHERE + software_for_community.status = 'pending' +GROUP BY + software_for_community.community +; +$$; + +-- Rejected software count by community +-- BY DEFAULT we return count of approved software +-- IF public is FALSE we return total count (as far as RLS allows) +CREATE FUNCTION rejected_count_by_community() RETURNS TABLE ( + community UUID, + rejected_cnt BIGINT +) LANGUAGE sql STABLE AS +$$ +SELECT + software_for_community.community, + COUNT(DISTINCT software_for_community.software) AS rejected_cnt +FROM + software_for_community +WHERE + software_for_community.status = 'rejected' +GROUP BY + software_for_community.community +; +$$; + +-- Keywords with the count used by keyword settings +-- to show existing keywords with the count +CREATE FUNCTION keyword_count_for_community() RETURNS TABLE ( + id UUID, + keyword CITEXT, + cnt BIGINT +) LANGUAGE sql STABLE AS +$$ + SELECT + keyword.id, + keyword.value AS keyword, + keyword_count.cnt + FROM + keyword + LEFT JOIN + (SELECT + keyword_for_community.keyword, + COUNT(keyword_for_community.keyword) AS cnt + FROM + keyword_for_community + GROUP BY keyword_for_community.keyword + ) AS keyword_count ON keyword.id = keyword_count.keyword; +$$; + +-- Keywords by community +-- for editing keywords of specific community +CREATE FUNCTION keywords_by_community() RETURNS TABLE ( + id UUID, + keyword CITEXT, + community UUID +) LANGUAGE sql STABLE AS +$$ + SELECT + keyword.id, + keyword.value AS keyword, + keyword_for_community.community + FROM + keyword_for_community + INNER JOIN + keyword ON keyword.id = keyword_for_community.keyword; +$$; +-- using filter ?community=eq.UUID + +-- Keywords grouped by community for filtering +-- We use array for selecting community with specific keywords +-- We use text value for "wild card" search +CREATE FUNCTION keyword_filter_for_community() RETURNS TABLE ( + community UUID, + keywords CITEXT[], + keywords_text TEXT +) LANGUAGE sql STABLE AS +$$ + SELECT + keyword_for_community.community AS community, + ARRAY_AGG( + keyword.value + ORDER BY value + ) AS keywords, + STRING_AGG( + keyword.value,' ' + ORDER BY value + ) AS keywords_text + FROM + keyword_for_community + INNER JOIN + keyword ON keyword.id = keyword_for_community.keyword + GROUP BY keyword_for_community.community; +$$; + + +-- rpc for community overview page +-- BY DEFAULT we return count of approved software +-- IF public is FALSE we return total count (as far as RLS allows) +CREATE FUNCTION communities_overview(public BOOLEAN DEFAULT TRUE) RETURNS TABLE ( + id UUID, + slug VARCHAR, + name VARCHAR, + short_description VARCHAR, + logo_id VARCHAR, + primary_maintainer UUID, + software_cnt BIGINT, + pending_cnt BIGINT, + rejected_cnt BIGINT, + keywords CITEXT[], + description VARCHAR, + created_at TIMESTAMPTZ +) LANGUAGE sql STABLE AS +$$ +SELECT + community.id, + community.slug, + community."name", + community.short_description, + community.logo_id, + community.primary_maintainer, + COALESCE(software_count_by_community.software_cnt, 0), + COALESCE(pending_count_by_community.pending_cnt, 0), + COALESCE(rejected_count_by_community.rejected_cnt, 0), + keyword_filter_for_community.keywords, + community.description, + community.created_at +FROM + community +LEFT JOIN + software_count_by_community(public) ON community.id = software_count_by_community.community +LEFT JOIN + pending_count_by_community() ON community.id = pending_count_by_community.community +LEFT JOIN + rejected_count_by_community() ON community.id = rejected_count_by_community.community +LEFT JOIN + keyword_filter_for_community() ON community.id=keyword_filter_for_community.community +; +$$; + +-- CATEGORIES for software of specific community +CREATE FUNCTION com_software_categories(community_id UUID) RETURNS TABLE( + software UUID, + category VARCHAR[] +) LANGUAGE sql STABLE AS +$$ + SELECT + category_for_software.software_id AS software, + ARRAY_AGG( + DISTINCT category_path.short_name + ORDER BY category_path.short_name + ) AS category + FROM + category_for_software + INNER JOIN + category_path(category_for_software.category_id) ON TRUE + WHERE + category_path.community = community_id + GROUP BY + category_for_software.software_id; +$$; + +-- SOFTWARE info by community +-- we filter this view at least by community_id (uuid) +CREATE FUNCTION software_by_community(community_id UUID) RETURNS TABLE ( + id UUID, + slug VARCHAR, + brand_name VARCHAR, + short_statement VARCHAR, + image_id VARCHAR, + is_published BOOLEAN, + updated_at TIMESTAMPTZ, + status request_status, + keywords CITEXT[], + prog_lang TEXT[], + licenses VARCHAR[], + categories VARCHAR[], + contributor_cnt BIGINT, + mention_cnt BIGINT +) LANGUAGE sql STABLE AS +$$ +SELECT DISTINCT ON (software.id) + software.id, + software.slug, + software.brand_name, + software.short_statement, + software.image_id, + software.is_published, + software.updated_at, + software_for_community.status, + keyword_filter_for_software.keywords, + prog_lang_filter_for_software.prog_lang, + license_filter_for_software.licenses, + com_software_categories.category AS categories, + count_software_contributors.contributor_cnt, + count_software_mentions.mention_cnt +FROM + software +LEFT JOIN + software_for_community ON software.id=software_for_community.software +LEFT JOIN + count_software_contributors() ON software.id=count_software_contributors.software +LEFT JOIN + count_software_mentions() ON software.id=count_software_mentions.software +LEFT JOIN + keyword_filter_for_software() ON software.id=keyword_filter_for_software.software +LEFT JOIN + prog_lang_filter_for_software() ON software.id=prog_lang_filter_for_software.software +LEFT JOIN + license_filter_for_software() ON software.id=license_filter_for_software.software +LEFT JOIN + com_software_categories(community_id) ON software.id=com_software_categories.software +WHERE + software_for_community.community = community_id +; +$$; + +-- SOFTWARE OF COMMUNITY LIST FOR SEARCH +-- WITH keywords, programming languages and licenses for filtering +CREATE FUNCTION software_by_community_search( + community_id UUID, + search VARCHAR +) RETURNS TABLE ( + id UUID, + slug VARCHAR, + brand_name VARCHAR, + short_statement VARCHAR, + image_id VARCHAR, + is_published BOOLEAN, + updated_at TIMESTAMPTZ, + status request_status, + keywords CITEXT[], + prog_lang TEXT[], + licenses VARCHAR[], + categories VARCHAR[], + contributor_cnt BIGINT, + mention_cnt BIGINT +) LANGUAGE sql STABLE AS +$$ +SELECT DISTINCT ON (software.id) + software.id, + software.slug, + software.brand_name, + software.short_statement, + software.image_id, + software.is_published, + software.updated_at, + software_for_community.status, + keyword_filter_for_software.keywords, + prog_lang_filter_for_software.prog_lang, + license_filter_for_software.licenses, + com_software_categories.category AS categories, + count_software_contributors.contributor_cnt, + count_software_mentions.mention_cnt +FROM + software +LEFT JOIN + software_for_community ON software.id=software_for_community.software +LEFT JOIN + count_software_contributors() ON software.id=count_software_contributors.software +LEFT JOIN + count_software_mentions() ON software.id=count_software_mentions.software +LEFT JOIN + keyword_filter_for_software() ON software.id=keyword_filter_for_software.software +LEFT JOIN + prog_lang_filter_for_software() ON software.id=prog_lang_filter_for_software.software +LEFT JOIN + license_filter_for_software() ON software.id=license_filter_for_software.software +LEFT JOIN + com_software_categories(community_id) ON software.id=com_software_categories.software +WHERE + software_for_community.community = community_id AND ( + software.brand_name ILIKE CONCAT('%', search, '%') + OR + software.slug ILIKE CONCAT('%', search, '%') + OR + software.short_statement ILIKE CONCAT('%', search, '%') + OR + keyword_filter_for_software.keywords_text ILIKE CONCAT('%', search, '%') + ) +ORDER BY + software.id, + CASE + WHEN brand_name ILIKE search THEN 0 + WHEN brand_name ILIKE CONCAT(search, '%') THEN 1 + WHEN brand_name ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END, + CASE + WHEN slug ILIKE search THEN 0 + WHEN slug ILIKE CONCAT(search, '%') THEN 1 + WHEN slug ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END, + CASE + WHEN short_statement ILIKE search THEN 0 + WHEN short_statement ILIKE CONCAT(search, '%') THEN 1 + WHEN short_statement ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END +; +$$; + +-- REACTIVE KEYWORD FILTER WITH COUNTS FOR SOFTWARE +-- PROVIDES AVAILABLE KEYWORDS FOR APPLIED FILTERS +CREATE FUNCTION com_software_keywords_filter( + community_id UUID, + software_status request_status DEFAULT 'approved', + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + keyword CITEXT, + keyword_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(keywords) AS keyword, + COUNT(id) AS keyword_cnt +FROM + software_by_community_search(community_id,search_filter) +WHERE + software_by_community_search.status = software_status + AND + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + COALESCE(categories, '{}') @> category_filter +GROUP BY + keyword +; +$$; + +-- REACTIVE PROGRAMMING LANGUAGES WITH COUNTS FOR SOFTWARE +-- PROVIDES AVAILABLE PROGRAMMING LANGUAGES FOR APPLIED FILTERS +CREATE FUNCTION com_software_languages_filter( + community_id UUID, + software_status request_status DEFAULT 'approved', + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + prog_language TEXT, + prog_language_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(prog_lang) AS prog_language, + COUNT(id) AS prog_language_cnt +FROM + software_by_community_search(community_id,search_filter) +WHERE + software_by_community_search.status = software_status + AND + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + COALESCE(categories, '{}') @> category_filter +GROUP BY + prog_language +; +$$; + +-- REACTIVE LICENSES FILTER WITH COUNTS FOR SOFTWARE +-- PROVIDES AVAILABLE LICENSES FOR APPLIED FILTERS +CREATE FUNCTION com_software_licenses_filter( + community_id UUID, + software_status request_status DEFAULT 'approved', + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + license VARCHAR, + license_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(licenses) AS license, + COUNT(id) AS license_cnt +FROM + software_by_community_search(community_id,search_filter) +WHERE + software_by_community_search.status = software_status + AND + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + COALESCE(categories, '{}') @> category_filter +GROUP BY + license +; +$$; + +-- REACTIVE CATEGORIES FILTER WITH COUNTS FOR SOFTWARE +-- PROVIDES AVAILABLE CATEGORIES FOR APPLIED FILTERS +CREATE FUNCTION com_software_categories_filter( + community_id UUID, + software_status request_status DEFAULT 'approved', + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + category VARCHAR, + category_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(categories) AS category, + -- count per software on unique software id + COUNT(DISTINCT(id)) AS category_cnt +FROM + software_by_community_search(community_id,search_filter) +WHERE + software_by_community_search.status = software_status + AND + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + COALESCE(categories, '{}') @> category_filter +GROUP BY + category +; +$$; + +-- GET COMMUNITIES OF SPECIFIC SOFTWARE +-- for software-edit-section +CREATE FUNCTION communities_of_software(software_id UUID) RETURNS TABLE ( + id UUID, + slug VARCHAR, + status request_status, + name VARCHAR, + short_description VARCHAR, + logo_id VARCHAR, + primary_maintainer UUID, + software_cnt BIGINT, + pending_cnt BIGINT, + rejected_cnt BIGINT, + keywords CITEXT[], + description VARCHAR, + created_at TIMESTAMPTZ +) LANGUAGE sql STABLE AS +$$ +SELECT + community.id, + community.slug, + software_for_community.status, + community."name", + community.short_description, + community.logo_id, + community.primary_maintainer, + software_count_by_community.software_cnt, + pending_count_by_community.pending_cnt, + rejected_count_by_community.rejected_cnt, + keyword_filter_for_community.keywords, + community.description, + community.created_at +FROM + community +LEFT JOIN + software_count_by_community() ON community.id = software_count_by_community.community +LEFT JOIN + pending_count_by_community() ON community.id = pending_count_by_community.community +LEFT JOIN + rejected_count_by_community() ON community.id = rejected_count_by_community.community +LEFT JOIN + keyword_filter_for_community() ON community.id = keyword_filter_for_community.community +INNER JOIN + software_for_community ON community.id = software_for_community.community +WHERE + software_for_community.software = software_id +; +$$; + + +-- GET COMMUNITIES FOR SPECIFIC USER/MAINTAINER +-- for software-edit-section +CREATE FUNCTION communities_by_maintainer(maintainer_id UUID) RETURNS TABLE ( + id UUID, + slug VARCHAR, + name VARCHAR, + short_description VARCHAR, + logo_id VARCHAR, + primary_maintainer UUID, + software_cnt BIGINT, + pending_cnt BIGINT, + rejected_cnt BIGINT, + keywords CITEXT[], + description VARCHAR, + created_at TIMESTAMPTZ +) LANGUAGE sql STABLE AS +$$ +SELECT DISTINCT ON (community.id) + community.id, + community.slug, + community."name", + community.short_description, + community.logo_id, + community.primary_maintainer, + COALESCE(software_count_by_community.software_cnt, 0), + COALESCE(pending_count_by_community.pending_cnt, 0), + COALESCE(rejected_count_by_community.rejected_cnt, 0), + keyword_filter_for_community.keywords, + community.description, + community.created_at +FROM + community +LEFT JOIN + software_count_by_community() ON community.id = software_count_by_community.community +LEFT JOIN + pending_count_by_community() ON community.id = pending_count_by_community.community +LEFT JOIN + rejected_count_by_community() ON community.id = rejected_count_by_community.community +LEFT JOIN + keyword_filter_for_community() ON community.id = keyword_filter_for_community.community +LEFT JOIN + maintainer_for_community ON maintainer_for_community.community = community.id +WHERE + maintainer_for_community.maintainer = maintainer_id OR community.primary_maintainer = maintainer_id; +; +$$; diff --git a/database/109-category-functions.sql b/database/109-category-functions.sql new file mode 100644 index 0000000..419c719 --- /dev/null +++ b/database/109-category-functions.sql @@ -0,0 +1,83 @@ +-- SPDX-FileCopyrightText: 2024 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2024 Netherlands eScience Center +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- DELETE organisation categories for specific community +CREATE FUNCTION delete_community_categories_from_software(software_id UUID, community_id UUID) +RETURNS VOID +LANGUAGE sql AS +$$ +DELETE FROM category_for_software + USING + category + WHERE + category_for_software.category_id = category.id AND + category_for_software.software_id = delete_community_categories_from_software.software_id AND + category.community = delete_community_categories_from_software.community_id; +$$; + + +-- DELETE organisation categories for specific software +CREATE FUNCTION delete_organisation_categories_from_software(software_id UUID, organisation_id UUID) +RETURNS VOID +LANGUAGE sql AS +$$ +DELETE FROM category_for_software + USING + category + WHERE + category_for_software.category_id = category.id AND + category_for_software.software_id = delete_organisation_categories_from_software.software_id AND + category.organisation = delete_organisation_categories_from_software.organisation_id; +$$; + +-- DELETE organisation categories for specific project +CREATE FUNCTION delete_organisation_categories_from_project(project_id UUID, organisation_id UUID) +RETURNS VOID +LANGUAGE sql AS +$$ +DELETE FROM category_for_project + USING + category + WHERE + category_for_project.category_id = category.id AND + category_for_project.project_id = delete_organisation_categories_from_project.project_id AND + category.organisation = delete_organisation_categories_from_project.organisation_id; +$$; + +CREATE FUNCTION delete_category_node(category_id UUID) + RETURNS VOID + LANGUAGE plpgsql + SECURITY DEFINER + VOLATILE +AS +$$ +DECLARE child_id UUID; +DECLARE child_ids UUID[]; +BEGIN + IF category_id IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please provide the ID of the category to delete'; + END IF; + + IF + (SELECT rolsuper FROM pg_roles WHERE rolname = SESSION_USER) IS DISTINCT FROM TRUE + AND + (SELECT CURRENT_SETTING('request.jwt.claims', FALSE)::json->>'role') IS DISTINCT FROM 'rsd_admin' + THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to delete this category'; + END IF; + + child_ids := (SELECT COALESCE((SELECT ARRAY_AGG(category.id) FROM category WHERE category.parent = delete_category_node.category_id), '{}')); + + FOREACH child_id IN ARRAY child_ids LOOP + PERFORM delete_category_node(child_id); + END LOOP; + + DELETE FROM category_for_software WHERE category_for_software.category_id = delete_category_node.category_id; + DELETE FROM category_for_project WHERE category_for_project.category_id = delete_category_node.category_id; + + DELETE FROM category WHERE category.id = delete_category_node.category_id; +END; +$$; diff --git a/database/113-organisation-views.sql b/database/113-organisation-views.sql new file mode 100644 index 0000000..313f9ad --- /dev/null +++ b/database/113-organisation-views.sql @@ -0,0 +1,728 @@ +-- SPDX-FileCopyrightText: 2023 - 2025 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center +-- SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE FUNCTION delete_organisation(id UUID) +RETURNS VOID +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +AS +$$ +DECLARE child_id UUID; +DECLARE child_ids UUID[]; +DECLARE category_id UUID; +DECLARE category_ids UUID[]; +BEGIN + IF id IS NULL THEN + RAISE EXCEPTION USING MESSAGE = 'Please provide the ID of the organisation to delete'; + END IF; + + IF + (SELECT rolsuper FROM pg_roles WHERE rolname = SESSION_USER) IS DISTINCT FROM TRUE + AND + (SELECT CURRENT_SETTING('request.jwt.claims', FALSE)::json->>'role') IS DISTINCT FROM 'rsd_admin' + THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to delete this organisation'; + END IF; + + child_ids := (SELECT COALESCE((SELECT ARRAY_AGG(organisation.id) FROM organisation WHERE organisation.parent = delete_organisation.id), '{}')); + + FOREACH child_id IN ARRAY child_ids LOOP + PERFORM delete_organisation(child_id); + END LOOP; + + category_ids := (SELECT COALESCE((SELECT ARRAY_AGG(category.id) FROM category WHERE category.organisation = delete_organisation.id), '{}')); + + FOREACH category_id IN ARRAY category_ids LOOP + PERFORM delete_category_node(category_id); + END LOOP; + + DELETE FROM invite_maintainer_for_organisation WHERE invite_maintainer_for_organisation.organisation = delete_organisation.id; + DELETE FROM maintainer_for_organisation WHERE maintainer_for_organisation.organisation = delete_organisation.id; + DELETE FROM project_for_organisation WHERE project_for_organisation.organisation = delete_organisation.id; + DELETE FROM software_for_organisation WHERE software_for_organisation.organisation = delete_organisation.id; + + DELETE FROM organisation WHERE organisation.id = delete_organisation.id; +END +$$; + +-- CATEGORIES for projects of specific organisation +CREATE FUNCTION org_project_categories(organisation_id UUID) RETURNS TABLE( + project UUID, + category VARCHAR[] +) LANGUAGE sql STABLE AS +$$ + SELECT + category_for_project.project_id AS project, + ARRAY_AGG( + DISTINCT category_path.short_name + ORDER BY category_path.short_name + ) AS category + FROM + category_for_project + INNER JOIN + category_path(category_for_project.category_id) ON TRUE + WHERE + category_path.organisation = organisation_id + GROUP BY + category_for_project.project_id; +$$; + +-- Project info by organisation +-- we filter this view at least by organisation_id (uuid) +CREATE FUNCTION projects_by_organisation(organisation_id UUID) RETURNS TABLE ( + id UUID, + slug VARCHAR, + title VARCHAR, + subtitle VARCHAR, + date_start DATE, + date_end DATE, + updated_at TIMESTAMPTZ, + is_published BOOLEAN, + image_contain BOOLEAN, + image_id VARCHAR, + is_featured BOOLEAN, + status relation_status, + keywords citext[], + research_domain VARCHAR[], + participating_organisations VARCHAR[], + categories VARCHAR[], + impact_cnt INTEGER, + output_cnt INTEGER, + project_status VARCHAR(20) +) LANGUAGE sql STABLE AS +$$ +SELECT DISTINCT ON (project.id) + project.id, + project.slug, + project.title, + project.subtitle, + project.date_start, + project.date_end, + project.updated_at, + project.is_published, + project.image_contain, + project.image_id, + project_for_organisation.is_featured, + project_for_organisation.status, + keyword_filter_for_project.keywords, + research_domain_filter_for_project.research_domain, + project_participating_organisations.organisations AS participating_organisations, + org_project_categories.category AS categories, + COALESCE(count_project_impact.impact_cnt, 0) AS impact_cnt, + COALESCE(count_project_output.output_cnt, 0) AS output_cnt, + project_status.status +FROM + project +LEFT JOIN + project_for_organisation ON project.id = project_for_organisation.project +LEFT JOIN + keyword_filter_for_project() ON project.id=keyword_filter_for_project.project +LEFT JOIN + research_domain_filter_for_project() ON project.id=research_domain_filter_for_project.project +LEFT JOIN + project_participating_organisations() ON project.id=project_participating_organisations.project +LEFT JOIN + org_project_categories(organisation_id) ON project.id=org_project_categories.project +LEFT JOIN + count_project_impact() ON project.id = count_project_impact.project +LEFT JOIN + count_project_output() ON project.id = count_project_output.project +LEFT JOIN + project_status() ON project.id=project_status.project +WHERE + project_for_organisation.organisation IN ( + SELECT list_child_organisations.organisation_id FROM list_child_organisations(organisation_id) + ) +; +$$; + +-- PROJECT OVERVIEW LIST FOR SEARCH +-- WITH keywords, research domain and participating organisations for filtering +CREATE FUNCTION projects_by_organisation_search( + organisation_id UUID, + search VARCHAR +) RETURNS TABLE ( + id UUID, + slug VARCHAR, + title VARCHAR, + subtitle VARCHAR, + date_start DATE, + date_end DATE, + updated_at TIMESTAMPTZ, + is_published BOOLEAN, + image_contain BOOLEAN, + image_id VARCHAR, + is_featured BOOLEAN, + status relation_status, + keywords citext[], + research_domain VARCHAR[], + participating_organisations VARCHAR[], + categories VARCHAR[], + impact_cnt INTEGER, + output_cnt INTEGER, + project_status VARCHAR(20) +) LANGUAGE sql STABLE AS +$$ +SELECT DISTINCT ON (project.id) + project.id, + project.slug, + project.title, + project.subtitle, + project.date_start, + project.date_end, + project.updated_at, + project.is_published, + project.image_contain, + project.image_id, + project_for_organisation.is_featured, + project_for_organisation.status, + keyword_filter_for_project.keywords, + research_domain_filter_for_project.research_domain, + project_participating_organisations.organisations AS participating_organisations, + org_project_categories.category AS categories, + COALESCE(count_project_impact.impact_cnt, 0) AS impact_cnt, + COALESCE(count_project_output.output_cnt, 0) AS output_cnt, + project_status.status +FROM + project +LEFT JOIN + project_for_organisation ON project.id = project_for_organisation.project +LEFT JOIN + keyword_filter_for_project() ON project.id=keyword_filter_for_project.project +LEFT JOIN + research_domain_filter_for_project() ON project.id=research_domain_filter_for_project.project +LEFT JOIN + project_participating_organisations() ON project.id=project_participating_organisations.project +LEFT JOIN + org_project_categories(organisation_id) ON project.id=org_project_categories.project +LEFT JOIN + count_project_impact() ON project.id = count_project_impact.project +LEFT JOIN + count_project_output() ON project.id = count_project_output.project +LEFT JOIN + project_status() ON project.id=project_status.project +WHERE + project_for_organisation.organisation IN ( + SELECT list_child_organisations.organisation_id FROM list_child_organisations(organisation_id) + ) AND ( + project.title ILIKE CONCAT('%', search, '%') + OR + project.slug ILIKE CONCAT('%', search, '%') + OR + project.subtitle ILIKE CONCAT('%', search, '%') + OR + keyword_filter_for_project.keywords_text ILIKE CONCAT('%', search, '%') + OR + research_domain_filter_for_project.research_domain_text ILIKE CONCAT('%', search, '%') + ) +ORDER BY + project.id, + CASE + WHEN title ILIKE search THEN 0 + WHEN title ILIKE CONCAT(search, '%') THEN 1 + WHEN title ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END, + CASE + WHEN slug ILIKE search THEN 0 + WHEN slug ILIKE CONCAT(search, '%') THEN 1 + WHEN slug ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END, + CASE + WHEN subtitle ILIKE search THEN 0 + WHEN subtitle ILIKE CONCAT(search, '%') THEN 1 + WHEN subtitle ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END +; +$$; + +-- REACTIVE KEYWORD FILTER WITH COUNTS FOR PROJECTS +-- PROVIDES AVAILABLE KEYWORDS FOR APPLIED FILTERS +CREATE FUNCTION org_project_keywords_filter( + organisation_id UUID, + search_filter TEXT DEFAULT '', + status_filter VARCHAR DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + research_domain_filter VARCHAR[] DEFAULT '{}', + organisation_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + keyword CITEXT, + keyword_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(keywords) AS keyword, + COUNT(id) AS keyword_cnt +FROM + projects_by_organisation_search(organisation_id,search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(research_domain, '{}') @> research_domain_filter + AND + COALESCE(participating_organisations, '{}') @> organisation_filter + AND + COALESCE(categories, '{}') @> category_filter + AND + CASE + WHEN status_filter <> '' THEN project_status = status_filter + ELSE true + END +GROUP BY + keyword +; +$$; + +-- REACTIVE RESEARCH DOMAIN FILTER WITH COUNTS FOR PROJECTS +-- PROVIDES AVAILABLE DOMAINS FOR APPLIED FILTERS +CREATE FUNCTION org_project_domains_filter( + organisation_id UUID, + search_filter TEXT DEFAULT '', + status_filter VARCHAR DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + research_domain_filter VARCHAR[] DEFAULT '{}', + organisation_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + domain VARCHAR, + domain_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(research_domain) AS domain, + COUNT(id) AS domain_cnt +FROM + projects_by_organisation_search(organisation_id,search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(research_domain, '{}') @> research_domain_filter + AND + COALESCE(participating_organisations, '{}') @> organisation_filter + AND + COALESCE(categories, '{}') @> category_filter + AND + CASE + WHEN status_filter <> '' THEN project_status = status_filter + ELSE true + END +GROUP BY + domain +; +$$; + +-- REACTIVE PARTICIPATING ORGANISATIONS FILTER WITH COUNTS FOR PROJECTS +-- PROVIDES AVAILABLE DOMAINS FOR APPLIED FILTERS +CREATE FUNCTION org_project_participating_organisations_filter( + organisation_id UUID, + search_filter TEXT DEFAULT '', + status_filter VARCHAR DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + research_domain_filter VARCHAR[] DEFAULT '{}', + organisation_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + organisation VARCHAR, + organisation_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(participating_organisations) AS organisation, + COUNT(id) AS organisation_cnt +FROM + projects_by_organisation_search(organisation_id,search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(research_domain, '{}') @> research_domain_filter + AND + COALESCE(participating_organisations, '{}') @> organisation_filter + AND + COALESCE(categories, '{}') @> category_filter + AND + CASE + WHEN status_filter <> '' THEN project_status = status_filter + ELSE true + END +GROUP BY + organisation +; +$$; + +-- REACTIVE PROJECT STATUS WITH COUNTS +-- PROVIDES AVAILABLE DOMAINS FOR APPLIED FILTERS +CREATE FUNCTION org_project_status_filter( + organisation_id UUID, + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + research_domain_filter VARCHAR[] DEFAULT '{}', + organisation_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + project_status VARCHAR, + project_status_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + project_status, + COUNT(id) AS project_status_cnt +FROM + projects_by_organisation_search(organisation_id,search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(research_domain, '{}') @> research_domain_filter + AND + COALESCE(participating_organisations, '{}') @> organisation_filter + AND + COALESCE(categories, '{}') @> category_filter +GROUP BY + project_status +; +$$; + +-- REACTIVE ORGANISATION CATEGORIES FILTER WITH COUNTS FOR PROJECTS +-- PROVIDES AVAILABLE CATEGORIES FOR APPLIED FILTERS +CREATE FUNCTION org_project_categories_filter( + organisation_id UUID, + search_filter TEXT DEFAULT '', + status_filter VARCHAR DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + research_domain_filter VARCHAR[] DEFAULT '{}', + organisation_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + category VARCHAR, + category_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(categories) AS category, + -- count per project on unique project id + COUNT(DISTINCT(id)) AS category_cnt +FROM + projects_by_organisation_search(organisation_id,search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(research_domain, '{}') @> research_domain_filter + AND + COALESCE(participating_organisations, '{}') @> organisation_filter + AND + COALESCE(categories, '{}') @> category_filter + AND + CASE + WHEN status_filter <> '' THEN project_status = status_filter + ELSE true + END +GROUP BY + category +; +$$; + +-- CATEGORIES for software of specific organisation +CREATE FUNCTION org_software_categories(organisation_id UUID) RETURNS TABLE( + software UUID, + category VARCHAR[] +) LANGUAGE sql STABLE AS +$$ + SELECT + category_for_software.software_id AS software, + ARRAY_AGG( + DISTINCT category_path.short_name + ORDER BY category_path.short_name + ) AS category + FROM + category_for_software + INNER JOIN + category_path(category_for_software.category_id) ON TRUE + WHERE + category_path.organisation = organisation_id + GROUP BY + category_for_software.software_id; +$$; + +-- SOFTWARE info by organisation +-- we filter this view at least by organisation_id (uuid) +CREATE FUNCTION software_by_organisation(organisation_id UUID) RETURNS TABLE ( + id UUID, + slug VARCHAR, + brand_name VARCHAR, + short_statement VARCHAR, + image_id VARCHAR, + is_published BOOLEAN, + updated_at TIMESTAMPTZ, + is_featured BOOLEAN, + status relation_status, + keywords CITEXT[], + prog_lang TEXT[], + licenses VARCHAR[], + categories VARCHAR[], + contributor_cnt BIGINT, + mention_cnt BIGINT +) LANGUAGE sql STABLE AS +$$ +SELECT DISTINCT ON (software.id) + software.id, + software.slug, + software.brand_name, + software.short_statement, + software.image_id, + software.is_published, + software.updated_at, + software_for_organisation.is_featured, + software_for_organisation.status, + keyword_filter_for_software.keywords, + prog_lang_filter_for_software.prog_lang, + license_filter_for_software.licenses, + org_software_categories.category AS categories, + count_software_contributors.contributor_cnt, + count_software_mentions.mention_cnt +FROM + software +LEFT JOIN + software_for_organisation ON software.id=software_for_organisation.software +LEFT JOIN + count_software_contributors() ON software.id=count_software_contributors.software +LEFT JOIN + count_software_mentions() ON software.id=count_software_mentions.software +LEFT JOIN + keyword_filter_for_software() ON software.id=keyword_filter_for_software.software +LEFT JOIN + prog_lang_filter_for_software() ON software.id=prog_lang_filter_for_software.software +LEFT JOIN + license_filter_for_software() ON software.id=license_filter_for_software.software +LEFT JOIN + org_software_categories(organisation_id) ON software.id=org_software_categories.software +WHERE + software_for_organisation.organisation IN ( + SELECT list_child_organisations.organisation_id FROM list_child_organisations(organisation_id) + ) +; +$$; + +-- SOFTWARE OF ORGANISATION LIST FOR SEARCH +-- WITH keywords, programming languages and licenses for filtering +CREATE FUNCTION software_by_organisation_search( + organisation_id UUID, + search VARCHAR +) RETURNS TABLE ( + id UUID, + slug VARCHAR, + brand_name VARCHAR, + short_statement VARCHAR, + image_id VARCHAR, + is_published BOOLEAN, + updated_at TIMESTAMPTZ, + is_featured BOOLEAN, + status relation_status, + keywords CITEXT[], + prog_lang TEXT[], + licenses VARCHAR[], + categories VARCHAR[], + contributor_cnt BIGINT, + mention_cnt BIGINT +) LANGUAGE sql STABLE AS +$$ +SELECT DISTINCT ON (software.id) + software.id, + software.slug, + software.brand_name, + software.short_statement, + software.image_id, + software.is_published, + software.updated_at, + software_for_organisation.is_featured, + software_for_organisation.status, + keyword_filter_for_software.keywords, + prog_lang_filter_for_software.prog_lang, + license_filter_for_software.licenses, + org_software_categories.category AS categories, + count_software_contributors.contributor_cnt, + count_software_mentions.mention_cnt +FROM + software +LEFT JOIN + software_for_organisation ON software.id=software_for_organisation.software +LEFT JOIN + count_software_contributors() ON software.id=count_software_contributors.software +LEFT JOIN + count_software_mentions() ON software.id=count_software_mentions.software +LEFT JOIN + keyword_filter_for_software() ON software.id=keyword_filter_for_software.software +LEFT JOIN + prog_lang_filter_for_software() ON software.id=prog_lang_filter_for_software.software +LEFT JOIN + license_filter_for_software() ON software.id=license_filter_for_software.software +LEFT JOIN + org_software_categories(organisation_id) ON software.id=org_software_categories.software +WHERE + software_for_organisation.organisation IN ( + SELECT list_child_organisations.organisation_id FROM list_child_organisations(organisation_id) + ) AND ( + software.brand_name ILIKE CONCAT('%', search, '%') + OR + software.slug ILIKE CONCAT('%', search, '%') + OR + software.short_statement ILIKE CONCAT('%', search, '%') + OR + keyword_filter_for_software.keywords_text ILIKE CONCAT('%', search, '%') + ) +ORDER BY + software.id, + CASE + WHEN brand_name ILIKE search THEN 0 + WHEN brand_name ILIKE CONCAT(search, '%') THEN 1 + WHEN brand_name ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END, + CASE + WHEN slug ILIKE search THEN 0 + WHEN slug ILIKE CONCAT(search, '%') THEN 1 + WHEN slug ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END, + CASE + WHEN short_statement ILIKE search THEN 0 + WHEN short_statement ILIKE CONCAT(search, '%') THEN 1 + WHEN short_statement ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END +; +$$; + +-- REACTIVE KEYWORD FILTER WITH COUNTS FOR SOFTWARE +-- PROVIDES AVAILABLE KEYWORDS FOR APPLIED FILTERS +CREATE FUNCTION org_software_keywords_filter( + organisation_id UUID, + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + keyword CITEXT, + keyword_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(keywords) AS keyword, + COUNT(id) AS keyword_cnt +FROM + software_by_organisation_search(organisation_id,search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + COALESCE(categories, '{}') @> category_filter +GROUP BY + keyword +; +$$; + +-- REACTIVE PROGRAMMING LANGUAGES WITH COUNTS FOR SOFTWARE +-- PROVIDES AVAILABLE PROGRAMMING LANGUAGES FOR APPLIED FILTERS +CREATE FUNCTION org_software_languages_filter( + organisation_id UUID, + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + prog_language TEXT, + prog_language_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(prog_lang) AS prog_language, + COUNT(id) AS prog_language_cnt +FROM + software_by_organisation_search(organisation_id,search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + COALESCE(categories, '{}') @> category_filter +GROUP BY + prog_language +; +$$; + +-- REACTIVE LICENSES FILTER WITH COUNTS FOR SOFTWARE +-- PROVIDES AVAILABLE LICENSES FOR APPLIED FILTERS +CREATE FUNCTION org_software_licenses_filter( + organisation_id UUID, + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + license VARCHAR, + license_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(licenses) AS license, + COUNT(id) AS license_cnt +FROM + software_by_organisation_search(organisation_id,search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + COALESCE(categories, '{}') @> category_filter +GROUP BY + license +; +$$; + +-- REACTIVE LICENSES FILTER WITH COUNTS FOR SOFTWARE +-- PROVIDES AVAILABLE LICENSES FOR APPLIED FILTERS +CREATE FUNCTION org_software_categories_filter( + organisation_id UUID, + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + category VARCHAR, + category_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(categories) AS category, + -- count per software on unique software id + COUNT(DISTINCT(id)) AS category_cnt +FROM + software_by_organisation_search(organisation_id,search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + COALESCE(categories, '{}') @> category_filter +GROUP BY + category +; +$$; diff --git a/database/114-keyword-category-views.sql b/database/114-keyword-category-views.sql new file mode 100644 index 0000000..4e2c570 --- /dev/null +++ b/database/114-keyword-category-views.sql @@ -0,0 +1,74 @@ +-- SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2025 Netherlands eScience Center +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE FUNCTION category_for_software_status(software_id UUID, category_id UUID) + RETURNS VARCHAR + LANGUAGE SQL + STABLE +AS +$$ +WITH + category_data AS (SELECT organisation, community FROM category WHERE category.id = category_id) +SELECT + CASE + WHEN (SELECT organisation FROM category_data) IS NULL AND (SELECT community FROM category_data) IS NULL THEN 'global' + WHEN (SELECT organisation FROM category_data AS organisation_id) IS NOT NULL THEN (SELECT status FROM software_for_organisation WHERE software_for_organisation.software = software_id AND software_for_organisation.organisation = (SELECT organisation FROM category_data AS organisation_id))::VARCHAR + WHEN (SELECT community FROM category_data) IS NOT NULL THEN (SELECT status FROM software_for_community WHERE software_for_community.software = software_id AND software_for_community.community = (SELECT community FROM category_data))::VARCHAR + ELSE 'other' + END +$$; + + +-- RPC for software page to show all software categories +CREATE FUNCTION category_paths_by_software_expanded(software_id UUID) + RETURNS JSONB + LANGUAGE SQL STABLE AS +$$ +SELECT COALESCE(jsonb_agg(paths.content), jsonb_build_array())::JSONB FROM ( + SELECT ARRAY_AGG(rows) AS content FROM ( + SELECT category_for_software.category_id, category_path.*, category_for_software_status AS status + FROM category_for_software + INNER JOIN category_path(category_for_software.category_id) ON TRUE + INNER JOIN category_for_software_status(category_paths_by_software_expanded.software_id, category_path.id) ON TRUE + WHERE category_for_software.software_id = category_paths_by_software_expanded.software_id + ) AS rows + GROUP BY rows.category_id +) AS paths +$$; + + +CREATE FUNCTION category_for_project_status(project_id UUID, category_id UUID) + RETURNS VARCHAR + LANGUAGE SQL + STABLE +AS +$$ +WITH + category_data AS (SELECT organisation FROM category WHERE category.id = category_id) +SELECT + CASE + WHEN (SELECT organisation FROM category_data) IS NULL THEN 'global' + WHEN (SELECT organisation FROM category_data AS organisation_id) IS NOT NULL THEN (SELECT status FROM project_for_organisation WHERE project_for_organisation.project = project_id AND project_for_organisation.organisation = (SELECT organisation FROM category_data AS organisation_id) AND role = 'participating')::VARCHAR + ELSE 'other' + END +$$; + + +-- RPC for project page to show all project categories +CREATE FUNCTION category_paths_by_project_expanded(project_id UUID) + RETURNS JSONB + LANGUAGE SQL STABLE AS +$$ +SELECT COALESCE(jsonb_agg(paths.content), jsonb_build_array())::JSONB FROM ( + SELECT ARRAY_AGG(rows) AS content FROM ( + SELECT category_for_project.category_id, category_path.*, category_for_project_status AS status + FROM category_for_project + INNER JOIN category_path(category_for_project.category_id) ON TRUE + INNER JOIN category_for_project_status(category_paths_by_project_expanded.project_id, category_path.id) ON TRUE + WHERE category_for_project.project_id = category_paths_by_project_expanded.project_id + ) AS rows + GROUP BY rows.category_id +) AS paths +$$; diff --git a/database/118-backend-logs-views.sql b/database/118-backend-logs-views.sql new file mode 100644 index 0000000..b2038fe --- /dev/null +++ b/database/118-backend-logs-views.sql @@ -0,0 +1,60 @@ +-- SPDX-FileCopyrightText: 2023 - 2024 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE FUNCTION slug_from_log_reference( + table_name VARCHAR, + reference_id UUID +) +RETURNS VARCHAR +LANGUAGE sql STABLE AS +$$ +SELECT CASE + WHEN table_name = 'repository_url' THEN ( + SELECT + CONCAT('/software/', slug, '/edit/information') + FROM + software WHERE id = reference_id + ) + WHEN table_name = 'package_manager' THEN ( + SELECT + CONCAT('/software/', slug, '/edit/package-managers') + FROM + software + WHERE id = (SELECT software FROM package_manager WHERE id = reference_id)) + WHEN table_name = 'mention' AND reference_id IS NOT NULL THEN ( + SELECT + CONCAT('/api/v1/mention?id=eq.', reference_id) + ) + WHEN table_name = 'organisation' AND reference_id IS NOT NULL THEN ( + SELECT + CONCAT('/organisations/', slug, '?tab=settings') + FROM + organisation + WHERE id = reference_id + ) + END +$$; + +CREATE FUNCTION backend_log_view() RETURNS TABLE ( + id UUID, + service_name VARCHAR, + table_name VARCHAR, + reference_id UUID, + message VARCHAR, + stack_trace VARCHAR, + other_data JSONB, + created_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ, + slug VARCHAR +) +LANGUAGE sql STABLE AS +$$ +SELECT + *, + slug_from_log_reference(backend_log.table_name, backend_log.reference_id) +FROM + backend_log +$$; diff --git a/database/124-aggregated-software-views.sql b/database/124-aggregated-software-views.sql new file mode 100644 index 0000000..5ddd317 --- /dev/null +++ b/database/124-aggregated-software-views.sql @@ -0,0 +1,315 @@ +-- SPDX-FileCopyrightText: 2024 - 2025 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- RPC AGGREGATED SOFTWARE VIEWS +-- DEPENDS ON: software_overview, remote_software +CREATE FUNCTION aggregated_software_overview() RETURNS TABLE ( + id UUID, + rsd_host VARCHAR, + domain VARCHAR, + slug VARCHAR, + brand_name VARCHAR, + short_statement VARCHAR, + image_id VARCHAR, + updated_at TIMESTAMPTZ, + contributor_cnt BIGINT, + mention_cnt BIGINT, + is_published BOOLEAN, + keywords CITEXT[], + keywords_text TEXT, + prog_lang TEXT[], + licenses VARCHAR[], + categories VARCHAR[] +) LANGUAGE sql STABLE AS +$$ +SELECT + software_overview.id, + COALESCE((SELECT value FROM rsd_info WHERE KEY='remote_name'),NULL) AS rsd_host, + NULL AS domain, + software_overview.slug, + software_overview.brand_name, + software_overview.short_statement, + software_overview.image_id, + software_overview.updated_at, + software_overview.contributor_cnt, + software_overview.mention_cnt, + software_overview.is_published, + software_overview.keywords, + software_overview.keywords_text, + software_overview.prog_lang, + software_overview.licenses, + software_overview.categories +FROM + software_overview() +UNION ALL +SELECT + remote_software.id, + remote_rsd.label AS rsd_host, + remote_rsd.domain, + remote_software.slug, + remote_software.brand_name, + remote_software.short_statement, + remote_software.image_id, + remote_software.updated_at, + remote_software.contributor_cnt, + remote_software.mention_cnt, + remote_software.is_published, + remote_software.keywords, + remote_software.keywords_text, + remote_software.prog_lang, + remote_software.licenses, + -- WE DO NOT USE/SCRAPE categories from remotes + '{}' AS categories +FROM + remote_software +INNER JOIN + remote_rsd ON remote_rsd.id = remote_software.remote_rsd_id +; +$$; + +-- AGGREGATES SOFTWARE OVERVIEW LIST FOR SEARCH +-- DEPENDS ON: aggregated_software_overview +CREATE FUNCTION aggregated_software_search(search VARCHAR) RETURNS TABLE ( + id UUID, + rsd_host VARCHAR, + domain VARCHAR, + slug VARCHAR, + brand_name VARCHAR, + short_statement VARCHAR, + image_id VARCHAR, + updated_at TIMESTAMPTZ, + contributor_cnt BIGINT, + mention_cnt BIGINT, + is_published BOOLEAN, + keywords CITEXT[], + keywords_text TEXT, + prog_lang TEXT[], + licenses VARCHAR[], + categories VARCHAR[] +) LANGUAGE sql STABLE AS +$$ +SELECT + id, rsd_host, domain, slug, brand_name, short_statement, image_id, + updated_at, contributor_cnt, mention_cnt, is_published, keywords, + keywords_text, prog_lang, licenses, categories +FROM + aggregated_software_overview() +WHERE + aggregated_software_overview.brand_name ILIKE CONCAT('%', search, '%') + OR + aggregated_software_overview.slug ILIKE CONCAT('%', search, '%') + OR + aggregated_software_overview.short_statement ILIKE CONCAT('%', search, '%') + OR + aggregated_software_overview.keywords_text ILIKE CONCAT('%', search, '%') +ORDER BY + CASE + WHEN aggregated_software_overview.brand_name ILIKE search THEN 0 + WHEN aggregated_software_overview.brand_name ILIKE CONCAT(search, '%') THEN 1 + WHEN aggregated_software_overview.brand_name ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END, + CASE + WHEN aggregated_software_overview.slug ILIKE search THEN 0 + WHEN aggregated_software_overview.slug ILIKE CONCAT(search, '%') THEN 1 + WHEN aggregated_software_overview.slug ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END, + CASE + WHEN aggregated_software_overview.short_statement ILIKE search THEN 0 + WHEN aggregated_software_overview.short_statement ILIKE CONCAT(search, '%') THEN 1 + WHEN aggregated_software_overview.short_statement ILIKE CONCAT('%', search, '%') THEN 2 + ELSE 3 + END +; +$$; + +-- REACTIVE KEYWORD FILTER WITH COUNTS FOR SOFTWARE +-- DEPENDS ON: aggregated_software_search +CREATE FUNCTION aggregated_software_keywords_filter( + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}', + rsd_host_filter VARCHAR DEFAULT '' +) RETURNS TABLE ( + keyword CITEXT, + keyword_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(keywords) AS keyword, + COUNT(id) AS keyword_cnt +FROM + aggregated_software_search(search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + CASE WHEN COALESCE(category_filter, '{}') = '{}' THEN TRUE ELSE COALESCE(categories, '{}') @> category_filter END + AND + CASE + WHEN rsd_host_filter = '' THEN TRUE + WHEN rsd_host_filter IS NULL THEN rsd_host IS NULL + ELSE + rsd_host = rsd_host_filter + END +GROUP BY + keyword +; +$$; + +-- REACTIVE PROGRAMMING LANGUAGES WITH COUNTS FOR SOFTWARE +-- DEPENDS ON: aggregated_software_search +CREATE FUNCTION aggregated_software_languages_filter( + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}', + rsd_host_filter VARCHAR DEFAULT '' +) RETURNS TABLE ( + prog_language TEXT, + prog_language_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(prog_lang) AS prog_language, + COUNT(id) AS prog_language_cnt +FROM + aggregated_software_search(search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + CASE WHEN COALESCE(category_filter, '{}') = '{}' THEN TRUE ELSE COALESCE(categories, '{}') @> category_filter END + AND + CASE + WHEN rsd_host_filter = '' THEN TRUE + WHEN rsd_host_filter IS NULL THEN rsd_host IS NULL + ELSE + rsd_host = rsd_host_filter + END +GROUP BY + prog_language +; +$$; + +-- REACTIVE LICENSES FILTER WITH COUNTS FOR SOFTWARE +-- DEPENDS ON: aggregated_software_search +CREATE FUNCTION aggregated_software_licenses_filter( + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}', + rsd_host_filter VARCHAR DEFAULT '' +) RETURNS TABLE ( + license VARCHAR, + license_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(licenses) AS license, + COUNT(id) AS license_cnt +FROM + aggregated_software_search(search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + CASE WHEN COALESCE(category_filter, '{}') = '{}' THEN TRUE ELSE COALESCE(categories, '{}') @> category_filter END + AND + CASE + WHEN rsd_host_filter = '' THEN TRUE + WHEN rsd_host_filter IS NULL THEN rsd_host IS NULL + ELSE + rsd_host = rsd_host_filter + END +GROUP BY + license +; +$$; + +-- REACTIVE CATEGORIES FILTER WITH COUNTS FOR SOFTWARE +-- DEPENDS ON: aggregated_software_search +CREATE FUNCTION aggregated_software_categories_filter( + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}', + rsd_host_filter VARCHAR DEFAULT '' +) RETURNS TABLE ( + category VARCHAR, + category_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + UNNEST(categories) AS category, + COUNT(DISTINCT(id)) AS category_cnt +FROM + aggregated_software_search(search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + COALESCE(categories, '{}') @> category_filter + AND + CASE + WHEN rsd_host_filter = '' THEN TRUE + WHEN rsd_host_filter IS NULL THEN rsd_host IS NULL + ELSE + rsd_host = rsd_host_filter + END +GROUP BY + category +; +$$; + +-- REACTIVE SOURCE FILTER WITH COUNTS FOR SOFTWARE +-- DEPENDS ON: aggregated_software_search +CREATE FUNCTION aggregated_software_hosts_filter( + search_filter TEXT DEFAULT '', + keyword_filter CITEXT[] DEFAULT '{}', + prog_lang_filter TEXT[] DEFAULT '{}', + license_filter VARCHAR[] DEFAULT '{}', + category_filter VARCHAR[] DEFAULT '{}' +) RETURNS TABLE ( + rsd_host VARCHAR, + rsd_host_cnt INTEGER +) LANGUAGE sql STABLE AS +$$ +SELECT + rsd_host, + COUNT(id) AS rsd_host_cnt +FROM + aggregated_software_search(search_filter) +WHERE + COALESCE(keywords, '{}') @> keyword_filter + AND + COALESCE(prog_lang, '{}') @> prog_lang_filter + AND + COALESCE(licenses, '{}') @> license_filter + AND + CASE WHEN COALESCE(category_filter, '{}') = '{}' THEN TRUE ELSE COALESCE(categories, '{}') @> category_filter END +GROUP BY + rsd_host +; +$$; diff --git a/database/125-global-search.sql b/database/125-global-search.sql new file mode 100644 index 0000000..e2b21c7 --- /dev/null +++ b/database/125-global-search.sql @@ -0,0 +1,187 @@ +-- SPDX-FileCopyrightText: 2025 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2025 Netherlands eScience Center +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- returns 0 when any of the ror_names is an exact (ignoring case) match, -1 when not found as substring, the least index otherwise (1-based) +CREATE FUNCTION index_of_ror_query(query VARCHAR, organisation_id UUID) +RETURNS INTEGER +LANGUAGE plpgsql +STABLE +AS +$$ + DECLARE query_lower VARCHAR := LOWER(query); + DECLARE ror_names VARCHAR[]; + DECLARE ror_name VARCHAR; + DECLARE ror_name_lower VARCHAR; + DECLARE min_index INTEGER; + BEGIN + ror_names := (SELECT organisation.ror_names FROM organisation WHERE id = organisation_id); + IF ror_names IS NULL THEN + RETURN -1; + ELSE + FOREACH ror_name IN ARRAY ror_names LOOP + CONTINUE WHEN ror_name IS NULL; + ror_name_lower := LOWER(ror_name); + IF ror_name_lower = query_lower THEN + RETURN 0; + ELSIF POSITION(query_lower IN ror_name_lower) <> 0 THEN + min_index := LEAST(min_index, POSITION(query_lower IN ror_name_lower)); + END IF; + END LOOP; + END IF; + IF min_index IS NULL THEN + RETURN -1; + ELSE + RETURN min_index; + END IF; + END; +$$; + +-- GLOBAL SEARCH +-- depends on: aggregated_software_search, public_user_profile +CREATE FUNCTION global_search(query VARCHAR) RETURNS TABLE( + slug VARCHAR, + domain VARCHAR, + rsd_host VARCHAR, + name VARCHAR, + source TEXT, + is_published BOOLEAN, + rank INTEGER, + index_found INTEGER +) LANGUAGE sql STABLE AS +$$ + -- AGGREGATED SOFTWARE search + SELECT + aggregated_software_search.slug, + aggregated_software_search.domain, + aggregated_software_search.rsd_host, + aggregated_software_search.brand_name AS name, + 'software' AS "source", + aggregated_software_search.is_published, + (CASE + WHEN aggregated_software_search.slug ILIKE query OR aggregated_software_search.brand_name ILIKE query THEN 0 + WHEN aggregated_software_search.keywords_text ILIKE CONCAT('%', query, '%') THEN 1 + WHEN aggregated_software_search.slug ILIKE CONCAT(query, '%') OR aggregated_software_search.brand_name ILIKE CONCAT(query, '%') THEN 2 + WHEN aggregated_software_search.slug ILIKE CONCAT('%', query, '%') OR aggregated_software_search.brand_name ILIKE CONCAT('%', query, '%') THEN 3 + ELSE 4 + END) AS rank, + (CASE + WHEN aggregated_software_search.slug ILIKE query OR aggregated_software_search.brand_name ILIKE query THEN 0 + WHEN aggregated_software_search.keywords_text ILIKE CONCAT('%', query, '%') THEN 0 + WHEN aggregated_software_search.slug ILIKE CONCAT(query, '%') OR aggregated_software_search.brand_name ILIKE CONCAT(query, '%') THEN 0 + WHEN aggregated_software_search.slug ILIKE CONCAT('%', query, '%') OR aggregated_software_search.brand_name ILIKE CONCAT('%', query, '%') + THEN LEAST(NULLIF(POSITION(LOWER(query) IN aggregated_software_search.slug), 0), NULLIF(POSITION(LOWER(query) IN LOWER(aggregated_software_search.brand_name)), 0)) + ELSE 0 + END) AS index_found + FROM + aggregated_software_search(query) + UNION ALL + -- PROJECT search + SELECT + project.slug, + NULL AS domain, + NULL as rsd_host, + project.title AS name, + 'projects' AS "source", + project.is_published, + (CASE + WHEN project.slug ILIKE query OR project.title ILIKE query THEN 0 + WHEN BOOL_OR(keyword.value ILIKE query) THEN 1 + WHEN project.slug ILIKE CONCAT(query, '%') OR project.title ILIKE CONCAT(query, '%') THEN 2 + WHEN project.slug ILIKE CONCAT('%', query, '%') OR project.title ILIKE CONCAT('%', query, '%') THEN 3 + ELSE 4 + END) AS rank, + (CASE + WHEN project.slug ILIKE query OR project.title ILIKE query THEN 0 + WHEN BOOL_OR(keyword.value ILIKE query) THEN 0 + WHEN project.slug ILIKE CONCAT(query, '%') OR project.title ILIKE CONCAT(query, '%') THEN 0 + WHEN project.slug ILIKE CONCAT('%', query, '%') OR project.title ILIKE CONCAT('%', query, '%') + THEN LEAST(NULLIF(POSITION(LOWER(query) IN project.slug), 0), NULLIF(POSITION(LOWER(query) IN LOWER(project.title)), 0)) + ELSE 0 + END) AS index_found + FROM + project + LEFT JOIN keyword_for_project ON keyword_for_project.project = project.id + LEFT JOIN keyword ON keyword.id = keyword_for_project.keyword + GROUP BY project.id + HAVING + project.slug ILIKE CONCAT('%', query, '%') + OR + project.title ILIKE CONCAT('%', query, '%') + OR + project.subtitle ILIKE CONCAT('%', query, '%') + OR + BOOL_OR(keyword.value ILIKE CONCAT('%', query, '%')) + UNION ALL + -- ORGANISATION search + SELECT + organisation.slug, + NULL AS domain, + NULL as rsd_host, + organisation."name", + 'organisations' AS "source", + TRUE AS is_published, + (CASE + WHEN organisation.slug ILIKE query OR organisation."name" ILIKE query OR index_of_ror_query(query, organisation.id) = 0 THEN 0 + WHEN organisation.slug ILIKE CONCAT(query, '%') OR organisation."name" ILIKE CONCAT(query, '%') OR index_of_ror_query(query, organisation.id) = 1 THEN 2 + ELSE 3 + END) AS rank, + (CASE + WHEN organisation.slug ILIKE query OR organisation."name" ILIKE query OR index_of_ror_query(query, organisation.id) = 0 THEN 0 + WHEN organisation.slug ILIKE CONCAT(query, '%') OR organisation."name" ILIKE CONCAT(query, '%') OR index_of_ror_query(query, organisation.id) = 1 THEN 0 + ELSE + LEAST(NULLIF(POSITION(LOWER(query) IN organisation.slug), 0), NULLIF(POSITION(LOWER(query) IN LOWER(organisation."name")), 0), NULLIF(index_of_ror_query(query, organisation.id), -1)) + END) AS index_found + FROM + organisation + WHERE + -- ONLY TOP LEVEL ORGANISATIONS + organisation.parent IS NULL + AND + (organisation.slug ILIKE CONCAT('%', query, '%') OR organisation."name" ILIKE CONCAT('%', query, '%') OR index_of_ror_query(query, organisation.id) >= 0) + UNION ALL + -- COMMUNITY search + SELECT + community.slug, + NULL AS domain, + NULL as rsd_host, + community."name", + 'communities' AS "source", + TRUE AS is_published, + (CASE + WHEN community.slug ILIKE query OR community."name" ILIKE query THEN 0 + WHEN community.slug ILIKE CONCAT(query, '%') OR community."name" ILIKE CONCAT(query, '%') THEN 2 + ELSE 3 + END) AS rank, + (CASE + WHEN community.slug ILIKE query OR community."name" ILIKE query THEN 0 + WHEN community.slug ILIKE CONCAT(query, '%') OR community."name" ILIKE CONCAT(query, '%') THEN 0 + ELSE + LEAST(NULLIF(POSITION(LOWER(query) IN community.slug), 0), NULLIF(POSITION(LOWER(query) IN LOWER(community."name")), 0)) + END) AS index_found + FROM + community + WHERE + community.slug ILIKE CONCAT('%', query, '%') OR community."name" ILIKE CONCAT('%', query, '%') + UNION ALL + -- PERSONS search + SELECT + CAST (public_user_profile.account AS VARCHAR) as slug, + NULL AS domain, + NULL as rsd_host, + public_user_profile.display_name as "name", + 'persons' AS "source", + public_user_profile.is_public AS is_published, + (CASE + WHEN public_user_profile.display_name ILIKE query THEN 0 + WHEN public_user_profile.display_name ILIKE CONCAT(query, '%') THEN 2 + ELSE 3 + END) AS rank, + 0 as index_found + FROM + public_user_profile() + WHERE + public_user_profile.display_name ILIKE CONCAT('%', query, '%'); +$$; diff --git a/database/126-public-persons-overview.sql b/database/126-public-persons-overview.sql new file mode 100644 index 0000000..c16117d --- /dev/null +++ b/database/126-public-persons-overview.sql @@ -0,0 +1,91 @@ +-- SPDX-FileCopyrightText: 2025 Dusan Mijatovic (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2025 Netherlands eScience Center +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- Software count per person +-- based on account id OR ORCID +-- acc_id = account, orc_id = ORCID +CREATE FUNCTION software_cnt_by_person(acc_id UUID, orc_id VARCHAR) RETURNS TABLE( + software_cnt BIGINT, + acc_id UUID, + orc_id VARCHAR +) LANGUAGE sql STABLE as +$$ +SELECT + count(*) AS software_cnt, + acc_id, + orc_id +FROM + software_by_public_profile() +WHERE + software_by_public_profile.orcid = orc_id + OR + software_by_public_profile.account = acc_id +$$; + +-- Project count per person +-- based on account id OR ORCID +-- acc_id = account, orc_id = ORCID +CREATE FUNCTION project_cnt_by_person(acc_id UUID, orc_id VARCHAR) RETURNS TABLE( + project_cnt BIGINT, + acc_id UUID, + orc_id VARCHAR +) LANGUAGE sql STABLE as +$$ +SELECT + count(*) AS project_cnt, + acc_id, + orc_id +FROM + project_by_public_profile() +WHERE + project_by_public_profile.orcid = orc_id + OR + project_by_public_profile.account = acc_id +$$; + +-- Public user profile information for profile page +-- this info overwrites aggregated contributors/team member info +-- from /rpc/person_mentions in 104-person-views +CREATE FUNCTION public_persons_overview() RETURNS TABLE ( + account UUID, + display_name VARCHAR, + affiliation VARCHAR, + "role" VARCHAR, + avatar_id VARCHAR(40), + orcid VARCHAR, + is_public BOOLEAN, + software_cnt BIGINT, + project_cnt BIGINT, + -- include keywords for future use + keywords VARCHAR[] +) LANGUAGE sql STABLE SECURITY DEFINER AS +$$ +SELECT + public_user_profile.account, + public_user_profile.display_name, + public_user_profile.affiliation, + public_user_profile.role, + public_user_profile.avatar_id, + public_user_profile.orcid, + public_user_profile.is_public, + software_cnt_by_person.software_cnt, + project_cnt_by_person.project_cnt, + -- include keywords for future use + array[]::varchar[] AS keywords +FROM + public_user_profile() +LEFT JOIN + software_cnt_by_person(public_user_profile.account,public_user_profile.orcid) ON ( + software_cnt_by_person.acc_id = public_user_profile.account + OR + software_cnt_by_person.orc_id = public_user_profile.orcid + ) +LEFT JOIN + project_cnt_by_person(public_user_profile.account,public_user_profile.orcid) ON ( + project_cnt_by_person.acc_id = public_user_profile.account + OR + project_cnt_by_person.orc_id = public_user_profile.orcid + ) +$$; diff --git a/database/201-nassa-import.sql b/database/201-nassa-import.sql new file mode 100644 index 0000000..56330b1 --- /dev/null +++ b/database/201-nassa-import.sql @@ -0,0 +1,239 @@ +-- SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +-- SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +-- +-- SPDX-License-Identifier: Apache-2.0 + +CREATE FUNCTION nassa_import( + slug VARCHAR(200), + brand_name VARCHAR(200), + description VARCHAR(10000), + short_statement VARCHAR(300), + get_started_url VARCHAR(200), + repository_url VARCHAR(200), + license_value VARCHAR(100), + license_name VARCHAR(200), + license_url VARCHAR(200), + license_open_source BOOLEAN, + related_modules VARCHAR[], + family_names_array VARCHAR[], + given_names_array VARCHAR[], + role_array VARCHAR[], + orcid_array VARCHAR[], + position_array INTEGER[], + categories JSONB, + regular_mentions JSONB +) +RETURNS VOID +LANGUAGE plpgsql +VOLATILE +AS +$$ +DECLARE software_id UUID; +DECLARE related_software_slug VARCHAR; +DECLARE related_software_id UUID; +DECLARE nassa_id UUID; +DECLARE category_value TEXT; +DECLARE category_id UUID; +DECLARE top_level_category_value TEXT; +DECLARE top_level_category_id UUID; +DECLARE mention_entry JSONB; +DECLARE mention_id UUID; + +BEGIN + IF + (SELECT rolsuper FROM pg_roles WHERE rolname = SESSION_USER) IS DISTINCT FROM TRUE + AND + (SELECT CURRENT_SETTING('request.jwt.claims', FALSE)::json->>'role') IS DISTINCT FROM 'rsd_admin' + THEN + RAISE EXCEPTION USING MESSAGE = 'You are not allowed to import NASSA software'; + END IF; + + SELECT id FROM community WHERE community.slug = 'nassa' INTO nassa_id; + IF nassa_id IS NULL + THEN + RAISE EXCEPTION USING MESSAGE = 'The NASSA community does not exist yet'; + END IF; + + IF + ARRAY_LENGTH(family_names_array, 1) IS DISTINCT FROM ARRAY_LENGTH(given_names_array, 1) + OR + ARRAY_LENGTH(given_names_array, 1) IS DISTINCT FROM ARRAY_LENGTH(role_array, 1) + OR + ARRAY_LENGTH(role_array, 1) IS DISTINCT FROM ARRAY_LENGTH(orcid_array, 1) + OR + ARRAY_LENGTH(orcid_array, 1) IS DISTINCT FROM ARRAY_LENGTH(position_array, 1) + THEN + RAISE EXCEPTION USING MESSAGE = 'The contributor arrays should have the same length'; + END IF; + + INSERT INTO software ( + slug, + is_published, + brand_name, + description, + short_statement, + get_started_url + ) VALUES ( + nassa_import.slug, + TRUE, + nassa_import.brand_name, + nassa_import.description, + nassa_import.short_statement, + nassa_import.get_started_url + ) + ON CONFLICT ((software.slug)) DO UPDATE SET + brand_name = EXCLUDED.brand_name, + description = EXCLUDED.description, + short_statement = EXCLUDED.short_statement, + get_started_url = EXCLUDED.get_started_url + RETURNING software.id INTO software_id; + + INSERT INTO software_for_community (software, community, status) VALUES (software_id, nassa_id, 'approved') ON CONFLICT DO NOTHING; + + INSERT INTO repository_url ( + software, + url, + code_platform, + scraping_disabled_reason + ) VALUES ( + software_id, + nassa_import.repository_url, + 'github', + 'This is a NASSA module which is not a repository root' + ) + ON CONFLICT DO NOTHING; + + DELETE FROM license_for_software WHERE license_for_software.software = software_id; + IF license_value IS NOT NULL + THEN + INSERT INTO license_for_software ( + software, + license, + name, + reference, + open_source + ) VALUES ( + software_id, + license_value, + license_name, + license_url, + license_open_source + ); + END IF; + + IF related_modules IS NOT NULL + THEN + FOREACH related_software_slug IN ARRAY related_modules LOOP + SELECT software.id FROM software WHERE software.slug = related_software_slug INTO related_software_id; + IF related_software_id IS NOT NULL + THEN + INSERT INTO software_for_software (origin, relation) VALUES (software_id, related_software_id) ON CONFLICT DO NOTHING; + END IF; + END LOOP; + END IF; + + -- contributors + FOR i IN 1..ARRAY_LENGTH(family_names_array, 1) LOOP + IF + orcid_array[i] IS NOT NULL + AND + (SELECT COUNT(*) FROM contributor WHERE contributor.software = software_id AND contributor.orcid = orcid_array[i]) = 1 + THEN + UPDATE contributor SET + family_names = family_names_array[i], + given_names = given_names_array[i], + role = role_array[i], + position = position_array[i] + WHERE + contributor.software = software_id AND contributor.orcid = orcid_array[i]; + ELSEIF (SELECT COUNT(*) FROM contributor WHERE contributor.software = software_id AND contributor.family_names = family_names_array[i] AND contributor.given_names = given_names_array[i]) = 1 + THEN + UPDATE contributor SET + role = role_array[i], + orcid = orcid_array[i], + position = position_array[i] + WHERE + contributor.software = software_id AND contributor.family_names = family_names_array[i] AND contributor.given_names = given_names_array[i]; + ELSE + INSERT INTO contributor ( + software, + family_names, + given_names, + role, + orcid, + position + ) VALUES ( + software_id, + family_names_array[i], + given_names_array[i], + role_array[i], + orcid_array[i], + position_array[i] + ); + END IF; + + END LOOP; + -- end contributors + + -- categories + FOR top_level_category_value IN (SELECT JSONB_OBJECT_KEYS(categories)) LOOP + SELECT id FROM category WHERE community = nassa_id AND name = top_level_category_value INTO top_level_category_id; + IF top_level_category_id IS NULL + THEN + INSERT INTO category ( + community, + short_name, + name + ) + VALUES ( + nassa_id, + top_level_category_value, + top_level_category_value + ) + RETURNING id INTO top_level_category_id; + END IF; + + FOR category_value IN (SELECT JSONB_ARRAY_ELEMENTS_TEXT(categories -> top_level_category_value)) LOOP + SELECT id FROM category WHERE community = nassa_id AND name = category_value INTO category_id; + IF category_id IS NULL + THEN + INSERT INTO category ( + community, + parent, + short_name, + name + ) + VALUES ( + nassa_id, + top_level_category_id, + category_value, + category_value + ) + RETURNING id INTO category_id; + END IF; + + INSERT INTO category_for_software (software_id, category_id) VALUES (software_id, category_id) ON CONFLICT DO NOTHING; + END LOOP; + END LOOP; + -- end categories + + -- regular mentions + FOR mention_entry IN (SELECT JSONB_ARRAY_ELEMENTS(regular_mentions)) LOOP + SELECT id FROM mention WHERE mention.doi = mention_entry ->> 'doi' INTO mention_id; + + IF mention_id IS NULL + THEN + SELECT id FROM mention WHERE mention.title = mention_entry ->> 'title' AND mention.authors IS NOT DISTINCT FROM mention_entry ->> 'authors' INTO mention_id; + END IF; + + IF mention_id IS NOT NULL + THEN + INSERT INTO mention_for_software (mention, software) VALUES (mention_id, software_id) ON CONFLICT DO NOTHING; + ELSE + INSERT INTO mention (SELECT * FROM JSONB_POPULATE_RECORD(NULL::mention, mention_entry)) RETURNING id INTO mention_id; + INSERT INTO mention_for_software (mention, software) VALUES (mention_id, software_id) ON CONFLICT DO NOTHING; + END IF; + END LOOP; + -- end regular mentions +END +$$; diff --git a/database/999-add-categories.sql.example b/database/999-add-categories.sql.example new file mode 100644 index 0000000..e043f4a --- /dev/null +++ b/database/999-add-categories.sql.example @@ -0,0 +1,102 @@ +-- SPDX-FileCopyrightText: 2023 Felix Mühlbauer (GFZ) +-- SPDX-FileCopyrightText: 2023 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- EXAMPLE FILE FOR SETTING UP SOME TEST CATEGORIES + +-- Remarks: +-- using icons: until #975 is addressed check/edit frontend/components/typography/Icon.tsx for available icons + +CREATE FUNCTION add_test_categories() +RETURNS TABLE (LIKE category) +LANGUAGE plpgsql +AS $$ + DECLARE parent1 uuid; + DECLARE parent2 uuid; + DECLARE parent3 uuid; +BEGIN + INSERT INTO category(short_name, name, properties) VALUES + ('Research fruits', 'Some simple research fruits', '{"icon":"science"}'::json) RETURNING id INTO parent1; + -- + INSERT INTO category(parent, short_name, name) VALUES + (parent1, 'Apple', 'Apple: An apple a day keeps...'), + (parent1, 'Bananas', 'Bananas, who bent the bananas?'), + (parent1, 'Grapes', 'Grapes, I like grapes.'), + (parent1, 'Pineapple', 'Pineapple is very sweet and tasty.'); + -- + -- + INSERT INTO category(short_name, name, properties) VALUES + ('A Highlighted Category', 'A Highlighted Category', '{"is_highlight":true, "description": "A description helps the user to understand the purpose of this category.", "subtitle": "A category could have a subtitle", "tree_level_labels": ["First person", "Number 2", "Other members"]}'::json) RETURNING id INTO parent1; + -- + INSERT INTO category(parent, short_name, name) VALUES + (parent1, '1 Ada', '1 Ada') RETURNING id INTO parent2; + INSERT INTO category(parent, short_name, name) VALUES + (parent2, '1.1 Ada, Bart', '1.1 Ada and Bart') RETURNING id INTO parent3; + INSERT INTO category(parent, short_name, name) VALUES + (parent3, '1.1.1 Ada, Bart, Chad', '1.1.1 Ada and Bart and Chad'); + INSERT INTO category(parent, short_name, name) VALUES + (parent3, '1.1.2 Ada, Bart, Carlos', '1.1.2 Ada and Bart and Carlos'); + -- + INSERT INTO category(parent, short_name, name) VALUES + (parent2, '1.2 Ada, Bob', '1.2 Ada and Bob') RETURNING id INTO parent3; + INSERT INTO category(parent, short_name, name) VALUES + (parent3, '1.2.1 Ada, Bob, Chad', '1.2.1 Ada and Bob and Chad'); + INSERT INTO category(parent, short_name, name) VALUES + (parent3, '1.2.2 Ada, Bob, Carlos', '1.2.2 Ada and Bob and Carlos'); + -- + INSERT INTO category(parent, short_name, name) VALUES + (parent1, '2 Alice', '2 Alice') RETURNING id INTO parent2; + INSERT INTO category(parent, short_name, name) VALUES + (parent2, '2.1 Alice, Bob', '2.1 Alice and Bob') RETURNING id INTO parent3; + INSERT INTO category(parent, short_name, name) VALUES + (parent3, '2.1.1 Alice, Bob, Carol', '2.1.1 Alice and Bob and Carol'); + INSERT INTO category(parent, short_name, name) VALUES + (parent3, '2.1.2 Alice, Bob, Charlie', '2.1.2 Alice and Bob and Charlie'); + -- + -- + INSERT INTO category(short_name, name) VALUES + ('Top level 1', 'Top Level Category 1') RETURNING id INTO parent1; + -- + INSERT INTO category(parent, short_name, name) VALUES + (parent1, 'Category A', 'Category A aka 1.1') RETURNING id INTO parent2; + INSERT INTO category(parent, short_name, name) VALUES + (parent2, 'Category 1 bit deeper', 'Category 1 bit deeper aka 1.1.1') RETURNING id INTO parent3; + -- + INSERT INTO category(parent, short_name, name) VALUES + (parent2, 'Category 2', 'Category 2 aka 1.1.2'), + (parent2, 'Category 3', 'Category 3 aka 1.1.3'), + -- + (parent3, 'Category a bit deeper', 'Category a bit deeper aka 1.1.1.1'), + (parent3, 'Category b bit deeper', 'Category b bit deeper aka 1.1.1.2'), + (parent3, 'Category c bit deeper and longer', 'Category c bit deeper and longer aka 1.1.1.3'), + (parent3, 'Category d bit deeper and longer', 'Category d bit deeper and longer aka 1.1.1.4'); + -- + INSERT INTO category(parent, short_name, name) VALUES + (parent1, 'Category B', 'Category B aka 1.2') RETURNING id INTO parent2; + INSERT INTO category(parent, short_name, name) VALUES + (parent2, 'Category 1', 'Category 1 aka 1.2.1'), + (parent2, 'Category 2', 'Category 2 aka 1.2.2'), + (parent2, 'Category 3', 'Category 3 aka 1.2.3'); + -- + -- + INSERT INTO category(short_name, name) VALUES + ('Top level 2 bit longer', 'Top level 2 bit longer aka 2') RETURNING id INTO parent1; + -- + INSERT INTO category(parent, short_name, name) VALUES + (parent1, 'Category A bit longer', 'Category A bit longer aka 2.1') RETURNING id INTO parent2; + INSERT INTO category(parent, short_name, name) VALUES + (parent2, 'Category 1 bit longer', 'Category 1 bit longer aka 2.1.1'), + (parent2, 'Category 2 bit longer', 'Category 2 bit longer aka 2.1.2'), + (parent2, 'Category 3 bit longer', 'Category 3 bit longer aka 2.1.3'); + -- + INSERT INTO category(parent, short_name, name) VALUES + (parent1, 'Category B even more longer', 'Category B even more longer aka 2.2') RETURNING id INTO parent2; + INSERT INTO category(parent, short_name, name) VALUES + (parent2, 'Category 1 even more longer', 'Category 1 even more longer aka 2.2.1'), + (parent2, 'Category 2 even more longer', 'Category 2 even more longer aka 2.2.2'), + (parent2, 'Category 3 even more longer', 'Category 3 even more longer aka 2.2.3'); +END +$$; + +SELECT add_test_categories(); diff --git a/database/Dockerfile b/database/Dockerfile new file mode 100644 index 0000000..e78605e --- /dev/null +++ b/database/Dockerfile @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2021 - 2025 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2021 - 2025 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +FROM postgres:15.12 +RUN chmod a+rwx /docker-entrypoint-initdb.d +COPY --chown=postgres:postgres *.sh /docker-entrypoint-initdb.d/ +COPY --chown=postgres:postgres *.sql /docker-entrypoint-initdb.d/ diff --git a/database/sonar-project.properties b/database/sonar-project.properties new file mode 100644 index 0000000..8749234 --- /dev/null +++ b/database/sonar-project.properties @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2023 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2023 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +sonar.projectKey=rsd-database +sonar.organization=research-software-directory + +# This is the name and version displayed in the SonarCloud UI. +#sonar.projectName=rsd-database +#sonar.projectVersion=1.0 + + +# Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. +#sonar.sources=. + +# Encoding of the source code. Default is default system encoding +#sonar.sourceEncoding=UTF-8 diff --git a/docker-compose.yml b/docker-compose.yml index 7651b0f..1959186 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,10 @@ services: database: - image: ghcr.io/research-software-directory/rsd-saas/database:v3.4.1 + build: ./database + image: kin-rpd/database:v1.1.2 + # initial rsd image version + # image: ghcr.io/research-software-directory/rsd-saas/database:v3.4.1 ports: # enable connection from outside (development mode) - "5432:5432" @@ -23,7 +26,10 @@ services: - net backend: - image: ghcr.io/research-software-directory/rsd-saas/backend:v3.4.1 + build: ./backend-postgrest + image: kin-rpd/backend:v1.1.2 + # initial rsd image version + # image: ghcr.io/research-software-directory/rsd-saas/backend:v3.4.1 expose: - 3500 environment: @@ -38,8 +44,28 @@ services: networks: - net + background-services: + build: ./backend-postgrest + image: kin-rpd/background-services:v1.1.2 + # initial rsd image version + # image: ghcr.io/research-software-directory/rsd-saas/background-services:v3.4.1 + environment: + # it uses values from .env file + - POSTGRES_DB_HOST + - POSTGRES_DB_HOST_PORT + - POSTGRES_DB + - POSTGRES_USER + - POSTGRES_PASSWORD + depends_on: + - database + networks: + - net + auth: - image: ghcr.io/research-software-directory/rsd-saas/auth:v3.4.1 + build: ./authentication + image: kin-rpd/auth:v1.1.2 + # initial rsd image version + # image: ghcr.io/research-software-directory/rsd-saas/auth:v3.4.1 ports: - 5005:5005 expose: @@ -92,7 +118,10 @@ services: ] scrapers: - image: ghcr.io/research-software-directory/rsd-saas/scrapers:v3.4.1 + build: ./scrapers + image: kin-rpd/scrapers:v1.1.2 + # initial rsd image version + # image: ghcr.io/research-software-directory/rsd-saas/scrapers:v3.4.1 environment: # it uses values from .env file - POSTGREST_URL @@ -111,20 +140,6 @@ services: networks: - net - background-services: - image: ghcr.io/research-software-directory/rsd-saas/background-services:v3.4.1 - environment: - # it uses values from .env file - - POSTGRES_DB_HOST - - POSTGRES_DB_HOST_PORT - - POSTGRES_DB - - POSTGRES_USER - - POSTGRES_PASSWORD - depends_on: - - database - networks: - - net - frontend: build: context: ./frontend diff --git a/scrapers/.editorconfig b/scrapers/.editorconfig new file mode 100644 index 0000000..76769f3 --- /dev/null +++ b/scrapers/.editorconfig @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2024 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +# EditorConfig is awesome: https://editorconfig.org + +# Unix-style newlines with a newline ending every file and tab indentation +[**] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true +indent_style = tab + +[**.md] +indent_style = unset + +[**.py] +indent_style = unset diff --git a/scrapers/Dockerfile b/scrapers/Dockerfile new file mode 100644 index 0000000..293db72 --- /dev/null +++ b/scrapers/Dockerfile @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +# SPDX-FileCopyrightText: 2022 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +# SPDX-FileCopyrightText: 2022 Matthias Rüster (GFZ) +# +# SPDX-License-Identifier: Apache-2.0 + +FROM maven:3.9.8-eclipse-temurin-21 AS builder +WORKDIR /usr/mymaven +RUN mkdir ./src +COPY pom.xml . +RUN mvn dependency:go-offline +COPY ./src ./src +RUN mvn package --offline + +FROM eclipse-temurin:21-jre-jammy +WORKDIR /usr/myjava +RUN apt-get update && apt-get --yes install cron nano +COPY jobs.cron /etc/cron.d/jobs.cron +RUN crontab /etc/cron.d/jobs.cron +COPY --from=builder /usr/mymaven/target/*-jar-with-dependencies.jar scrapers.jar +CMD printenv > /etc/environment && cron -f diff --git a/scrapers/README.md b/scrapers/README.md new file mode 100644 index 0000000..b6d0171 --- /dev/null +++ b/scrapers/README.md @@ -0,0 +1,36 @@ + + +# Scrapers + +[![Coverage](https://sonarcloud.io/api/project_badges/measure?project=nl.research-software%3Ascrapers&metric=coverage)](https://sonarcloud.io/summary/overall?id=nl.research-software%3Ascrapers) + +This module contains the scrapers used by the RSD. The scrapers are written in Java using Maven. + +## Contributing + +This module uses [EditorConfig](https://editorconfig.org/) for basic formatting. Please check if your editor [already supports EditorConfig](https://editorconfig.org/#pre-installed) or if you need to [install a plugin](https://editorconfig.org/#download). A GitHub workflow is run on every PR to check if any files violate the formatting settings. + +## Running from within Docker Compose + +If you have an instance of the RSD, including the scrapers, running with Docker Compose, you can manually run a scraper by running the respective command from the root of the project content: + +```shell +docker compose exec scrapers java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainProgrammingLanguages +docker compose exec scrapers java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainContributors +docker compose exec scrapers java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainBasicData +docker compose exec scrapers java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainCommits +docker compose exec scrapers java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.doi.MainMentions +docker compose exec scrapers java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.doi.MainReleases +docker compose exec scrapers java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.package_manager.MainPackageManager +docker compose exec scrapers java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.doi.MainCitations +docker compose exec scrapers java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.ror.MainRor +``` + +## Running locally + +To run a scraper locally, e.g. from an IDE, you need to make sure the values in `.env` are loaded and that the value for the env variable `POSTGREST_URL` is overwritten with an address that is reachable from outside of the Docker network (usually `http://localhost/api/v1`). diff --git a/scrapers/jobs.cron b/scrapers/jobs.cron new file mode 100644 index 0000000..6645377 --- /dev/null +++ b/scrapers/jobs.cron @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: 2021 - 2022 Netherlands eScience Center +# +# SPDX-License-Identifier: CC-BY-4.0 +# +*/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainProgrammingLanguages > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +*/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.package_manager.MainPackageManager > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +2-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainBasicData > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +4-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainCommits > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +1-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.doi.MainReleases > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +3-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.doi.MainMentions > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +5-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainContributors > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +5-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.doi.MainCitations > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +2-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.ror.MainRor > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +* * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.aggregator.MainAggregator > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +15 1 * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.nassa.MainNassa > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 diff --git a/scrapers/pom.xml b/scrapers/pom.xml new file mode 100644 index 0000000..243f979 --- /dev/null +++ b/scrapers/pom.xml @@ -0,0 +1,171 @@ + + + + + + + + 4.0.0 + + nl.research-software + scrapers + 1.0-SNAPSHOT + + + UTF-8 + 21 + 21 + research-software-directory + https://sonarcloud.io + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.7.1 + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.7.1 + + + + jar-with-dependencies + + + + + + package + + single + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.12 + + + prepare-agent + + prepare-agent + + + + report + + report + + + + XML + + + + + + + + + + + + org.junit + junit-bom + 5.10.3 + pom + import + + + + + + + + org.slf4j + slf4j-api + 2.0.13 + + + + + ch.qos.logback + logback-classic + 1.5.6 + + + + + com.auth0 + java-jwt + 4.5.0 + + + + + com.google.code.gson + gson + 2.11.0 + + + + + org.yaml + snakeyaml + 2.4 + + + + + + org.junit.jupiter + junit-jupiter + test + + + + + org.junit.platform + junit-platform-launcher + test + + + + + org.apache.maven.surefire + surefire-junit-platform + 3.2.5 + test + + + diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/BibtexParser.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/BibtexParser.java new file mode 100644 index 0000000..8278683 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/BibtexParser.java @@ -0,0 +1,335 @@ +// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper; + +import nl.esciencecenter.rsd.scraper.doi.Doi; +import nl.esciencecenter.rsd.scraper.doi.ExternalMentionRecord; +import nl.esciencecenter.rsd.scraper.doi.MentionType; + +import java.io.IOException; +import java.io.LineNumberReader; +import java.io.Reader; +import java.net.URI; +import java.util.Map; +import java.util.TreeMap; + +// https://www.bibtex.org/Format/ +// https://www.bibtex.com/e/entry-types/ +public class BibtexParser { + + static final Map BIBTEX_ENTRY_TO_MENTION_TYPE = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + + static { + BIBTEX_ENTRY_TO_MENTION_TYPE.put("article", MentionType.journalArticle); + BIBTEX_ENTRY_TO_MENTION_TYPE.put("book", MentionType.book); + BIBTEX_ENTRY_TO_MENTION_TYPE.put("booklet", MentionType.book); + BIBTEX_ENTRY_TO_MENTION_TYPE.put("conference", MentionType.conferencePaper); + BIBTEX_ENTRY_TO_MENTION_TYPE.put("inbook", MentionType.bookSection); + BIBTEX_ENTRY_TO_MENTION_TYPE.put("incollection", MentionType.bookSection); + BIBTEX_ENTRY_TO_MENTION_TYPE.put("inproceedings", MentionType.conferencePaper); + BIBTEX_ENTRY_TO_MENTION_TYPE.put("manual", MentionType.other); + BIBTEX_ENTRY_TO_MENTION_TYPE.put("mastersthesis", MentionType.thesis); + BIBTEX_ENTRY_TO_MENTION_TYPE.put("misc", MentionType.other); + BIBTEX_ENTRY_TO_MENTION_TYPE.put("phdthesis", MentionType.thesis); + BIBTEX_ENTRY_TO_MENTION_TYPE.put("proceedings", MentionType.book); + BIBTEX_ENTRY_TO_MENTION_TYPE.put("techreport", MentionType.other); + BIBTEX_ENTRY_TO_MENTION_TYPE.put("unpublished", MentionType.other); + } + + enum ParserState { + BETWEEN_ENTRIES, + READING_ENTRY_TYPE, + READING_ENTRY_OUTER_BRACE, + BEFORE_READING_CITEKEY, + READING_CITEKEY, + BEFORE_READING_KEY, + READING_KEY, + READING_EQUALS_SIGN, + READING_VALUE_DELIMITER, + READING_MONTH, + READING_INTEGER, + READING_VALUE, + READING_COMMA + } + + public static Map parse(Reader reader) throws IOException, InvalidInputException { + Map result = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + LineNumberReader lineNumberReader = new LineNumberReader(reader); + lineNumberReader.setLineNumber(1); + ParserState state = ParserState.BETWEEN_ENTRIES; + StringBuilder stringBuilder = new StringBuilder(); + int column = 0; + char valueDelimiter = 0; + int openBraceCount = 0; + + String entryType = null; + String citekey = null; + String key = null; + + Map keyValues = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + + while (true) { + int nextCharAsInt = lineNumberReader.read(); + if (nextCharAsInt == -1) { + return handleEndOfInput(result, state); + } + + char c = (char) nextCharAsInt; + if (c == '\n') { + column = 0; + } else { + ++column; + } + + switch (state) { + case BETWEEN_ENTRIES -> { + if (c == '@') { + state = ParserState.READING_ENTRY_TYPE; + stringBuilder.setLength(0); + } + // we accept all other character as comments (and ignore them) to accommodate the comma between the entries on line 7 from + // https://github.com/Archaeology-ABM/NASSA-modules/blob/main/2022-Verhagen-001/references.bib + } + case READING_ENTRY_TYPE -> { + if (Character.isWhitespace(c) || c == '{') { + if (stringBuilder.isEmpty()) { + throw new InvalidInputException("READING_ENTRY_TYPE Empty entry type at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + } + entryType = stringBuilder.toString(); + stringBuilder.setLength(0); + state = c == '{' ? ParserState.BEFORE_READING_CITEKEY : ParserState.READING_ENTRY_OUTER_BRACE; + } else if (!isAsciiLetter(c)) { + throw new InvalidInputException("READING_ENTRY_TYPE Expected a letter but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + } else { + stringBuilder.append(c); + } + } + case READING_ENTRY_OUTER_BRACE -> { + if (c == '{') { + state = ParserState.BEFORE_READING_KEY; + } else if (!Character.isWhitespace(c)) { + throw new InvalidInputException("READING_ENTRY_OUTER_BRACE Expected { or whitespace but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + } + } + case BEFORE_READING_CITEKEY -> { + if (!isValidCitekeyCharacter(c)) { + throw new InvalidInputException("BEFORE_READING_CITEKEY Expected an alphanumeric character or - or _ or : but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + } else if (!Character.isWhitespace(c)) { + stringBuilder.setLength(0); + stringBuilder.append(c); + state = ParserState.READING_CITEKEY; + } + } + case READING_CITEKEY -> { + if (isValidCitekeyCharacter(c)) { + stringBuilder.append(c); + } else if (c == ',') { + if (stringBuilder.isEmpty()) { + throw new InvalidInputException("READING_CITEKEY Empty citekey at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + } + citekey = stringBuilder.toString(); + stringBuilder.setLength(0); + state = ParserState.BEFORE_READING_KEY; + } else { + throw new InvalidInputException("READING_CITEKEY Expected an alphanumeric character or - or _ or : but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + } + } + case BEFORE_READING_KEY -> { + if (c == '}') { + state = ParserState.BETWEEN_ENTRIES; + handleEntry(keyValues, entryType, citekey, result); + } else if (isAsciiLetter(c)) { + stringBuilder.setLength(0); + stringBuilder.append(c); + state = ParserState.READING_KEY; + } else if (!Character.isWhitespace(c)) { + throw new InvalidInputException("BEFORE_READING_KEY Expected a letter but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + } + } + case READING_KEY -> { + // '-' needed to handle key mendeley-groups + if (isAsciiLetter(c) || c == '-') { + stringBuilder.append(c); + } else if (Character.isWhitespace(c) || c == '=') { + if (stringBuilder.isEmpty()) { + throw new InvalidInputException("READING_KEY Empty key at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + } + key = stringBuilder.toString(); + stringBuilder.setLength(0); + state = c == '=' ? ParserState.READING_VALUE_DELIMITER : ParserState.READING_EQUALS_SIGN; + } else { + throw new InvalidInputException("READING_KEY Expected a letter or whitespace or = but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + } + } + case READING_EQUALS_SIGN -> { + if (c == '=') { + state = ParserState.READING_VALUE_DELIMITER; + } else if (Character.isWhitespace(c)) { + throw new InvalidInputException("READING_EQUALS_SIGN Expected whitespace or = but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + } + } + case READING_VALUE_DELIMITER -> { + if (c == '{' || c == '"') { + valueDelimiter = c; + stringBuilder.setLength(0); + state = ParserState.READING_VALUE; + } else if (isAsciiLetter(c)) { + valueDelimiter = 0; + stringBuilder.setLength(0); + stringBuilder.append(c); + state = ParserState.READING_MONTH; + } else if (isAsciiDigit(c)) { + valueDelimiter = 0; + stringBuilder.setLength(0); + stringBuilder.append(c); + state = ParserState.READING_INTEGER; + } else if (!Character.isWhitespace(c)) { + throw new InvalidInputException("READING_VALUE_DELIMITER Expected whitespace or { or \" but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + } + } + case READING_MONTH -> { + if (isAsciiLetter(c)) { + if (stringBuilder.length() < 3) { + stringBuilder.append(c); + } else { + throw new InvalidInputException("READING_MONTH Month literal too long at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + } + } else if (Character.isWhitespace(c) || c == ',') { + if (stringBuilder.length() != 3) { + throw new InvalidInputException("READING_MONTH Month literal too short at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + } else { + keyValues.put(key, stringBuilder.toString()); + stringBuilder.setLength(0); + state = c == ',' ? ParserState.BEFORE_READING_KEY : ParserState.READING_COMMA; + } + } else { + throw new InvalidInputException("READING_MONTH Illegal month literal at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + } + } + case READING_INTEGER -> { + if (isAsciiDigit(c)) { + stringBuilder.append(c); + } else if (Character.isWhitespace(c) || c == ',') { + keyValues.put(key, stringBuilder.toString()); + stringBuilder.setLength(0); + state = c == ',' ? ParserState.BEFORE_READING_KEY : ParserState.READING_COMMA; + } else { + throw new InvalidInputException("READING_INTEGER Illegal integer literal at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + } + } + case READING_VALUE -> { + if (c == '{') { + ++openBraceCount; + } else if (c == '}') { + --openBraceCount; + if (openBraceCount < 0 && valueDelimiter != '{') { + throw new InvalidInputException("READING_VALUE Unmatched closing brace at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + } else if (openBraceCount < 0) { + keyValues.put(key, stringBuilder.toString()); + stringBuilder.setLength(0); + openBraceCount = 0; + state = ParserState.READING_COMMA; + } + } else if (c == '"' && valueDelimiter == '"') { + keyValues.put(key, stringBuilder.toString()); + stringBuilder.setLength(0); + state = ParserState.READING_COMMA; + } else { + stringBuilder.append(c); + } + } + case READING_COMMA -> { + if (c == ',') { + state = ParserState.BEFORE_READING_KEY; + } else if (c == '}') { + state = ParserState.BETWEEN_ENTRIES; + handleEntry(keyValues, entryType, citekey, result); + } else if (!Character.isWhitespace(c)) { + throw new InvalidInputException("READING_COMMA Expected whitespace or , or } but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + } + } + } + } + } + + static Map handleEndOfInput(Map result, ParserState state) throws InvalidInputException { + if (state == ParserState.BETWEEN_ENTRIES) { + return result; + } else { + throw new InvalidInputException("End of input reached while in state %s".formatted(state)); + } + } + + static boolean isValidCitekeyCharacter(char c) { + return (c >= 'a' && c <= 'z') + || + (c >= 'A' && c <= 'Z') + || + (c >= '0' && c <= '9') + || + c == '-' + || + c == '_' + || + c == ':'; + } + + static boolean isAsciiLetter(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + } + + static boolean isAsciiDigit(char c) { + return c >= '0' && c <= '9'; + } + + static void handleEntry( + Map keyValues, + String entryType, + String citekey, + Map mapToAddTo + ) { + String doiString = keyValues.get("doi"); + Doi doi = Doi.isValid(doiString) ? Doi.fromString(doiString) : null; + String urlString = keyValues.get("url"); + URI url = null; + if (doi != null) { + url = URI.create("https://doi.org/" + doi.toUrlEncodedString()); + } else if (urlString != null) { + url = URI.create(urlString); + } + String title = keyValues.get("title"); + String authors = keyValues.get("author"); + String publisher = keyValues.get("publisher"); + Integer publicationYear = null; + if (keyValues.get("year") != null) { + publicationYear = Integer.parseInt(keyValues.get("year")); + } + String journal = keyValues.get("journal"); + String page = keyValues.get("pages"); + MentionType mentionType = BIBTEX_ENTRY_TO_MENTION_TYPE.getOrDefault(entryType, MentionType.other); + String source = "NASSA"; + + ExternalMentionRecord recordToAdd = new ExternalMentionRecord( + doi, + null, + null, + url, + title, + authors, + publisher, + publicationYear, + journal, + page, + mentionType, + source, + null + ); + + mapToAddTo.put(citekey, recordToAdd); + keyValues.clear(); + } +} + diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Config.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Config.java new file mode 100644 index 0000000..87f9ca6 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Config.java @@ -0,0 +1,204 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +// SPDX-FileCopyrightText: 2022 dv4all +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Optional; + +/** + * Helper class to retrieve various environment variables. + */ +public class Config { + + private static final long TEN_MINUTES_IN_MILLISECONDS = 10L * 60L * 1000L; + + private static final Logger LOGGER = LoggerFactory.getLogger(Config.class); + + private Config() { + } + + /** + * Retrieves the value of the environment variable with the given name. Variable is not allowed to unset, and an Error + * will be thrown if this is the case. + * + * @param name the name of the variable. + * @return the value of the variable. + */ + private static String getMandatoryEnv(String name) { + + if (name == null || name.isBlank()) { + LOGGER.error("Attempting to retrieve mandatory environment variable without a name!"); + throw new Error("Attempting to retrieve mandatory environment variable without a name!"); + } + + String value; + + try { + value = System.getenv(name); + } catch (Exception e) { + LOGGER.error("Failed to retrieve environment variable: {}", name, e); + throw new Error("Failed to retrieve environment variable: " + name, e); + } + + if (value == null || value.isBlank()) { + LOGGER.error("Blank environment variable: {}", name); + throw new Error("Failed to retrieve environment variable: " + name); + } + + return value; + } + + /** + * Retrieves the value of the environment variable with the given name. Allows the variable to be unset. + * + * @param name the name of the variable. + * @return the value of the variable if set, or empty Optional otherwise. + */ + private static Optional getOptionalEnv(String name) { + + if (name == null || name.isBlank()) { + LOGGER.warn("Attempting to retrieve environment variable without a name!"); + return Optional.empty(); + } + + try { + String value = System.getenv(name); + return (value == null || value.isBlank()) ? Optional.empty() : Optional.of(value.strip()); + } catch (Exception e) { + LOGGER.warn("Failed to retrieve environment variable: {}", name, e); + return Optional.empty(); + } + } + + /** + * Retrieves the integer value of the environment variable with the given name. When the variable is unset, empty + * or contains an invalid value, the default value is returned. + * + * @param name the name of the variable. + * @param defaultValue the default value to return if the environment variable is unset, empty, or does not contain an integer. + * @return the value from the environment of the default value if no value could be obtained + */ + private static int getIntEnv(String name, int defaultValue) { + + if (name == null || name.isBlank()) { + LOGGER.warn("Attempting to retrieve environment variable without a name!"); + return defaultValue; + } + + try { + return Integer.parseInt(System.getenv(name)); + } catch (Exception e) { + LOGGER.warn("Failed to retrieve environment variable: {}", name, e); + } + + return defaultValue; + } + + /** + * Get the JWT expiration time (in milliseconds). + * + * @return the JWT expiration time. + */ + + public static long jwtExpirationTime() { + return TEN_MINUTES_IN_MILLISECONDS; + } + + /** + * Get Postgrest JWT secret. + * + * @return the Postgrest JWT secret. + */ + public static String jwtSigningSecret() { + return getMandatoryEnv("PGRST_JWT_SECRET"); + } + + /** + * Get base URL to connect to the backend. + * + * @return the Get base URL to connect to the backend. + */ + public static String backendBaseUrl() { + return getMandatoryEnv("POSTGREST_URL"); + } + + /** + * The maximum requests rate for GitHub. + * + * @return the maximum request rate (default 6). + */ + public static int maxRequestsGithub() { + return getIntEnv("MAX_REQUESTS_GITHUB", 6); + } + + /** + * The maximum requests rate for GitLab. + * + * @return the maximum request rate (default 6). + */ + public static int maxRequestsGitLab() { + return getIntEnv("MAX_REQUESTS_GITLAB", 6); + } + + /** + * Get the maximum number of mentions to scrape in one run of the MentionScraper. + * + * @return the maximum number of mentions to scrape (default 6). + */ + public static int maxRequestsDoi() { + return getIntEnv("MAX_REQUESTS_DOI", 6); + } + + public static int maxRequestsRor() { + return getIntEnv("MAX_REQUESTS_ROR", 6); + } + + /** + * Get the maximum number of citation sources to scrape in one run of the CitationScraper. + * + * @return the maximum number of citation sources to scrape (default 5). + */ + public static int maxCitationSourcesToScrape() { + return getIntEnv("MAX_REQUESTS_OPENALEX", 5); + } + + /** + * Get the email contact address for crossref. + * + * @return the Get the email contact address for crossref (default unset). + */ + public static Optional crossrefContactEmail() { + return getOptionalEnv("CROSSREF_CONTACT_EMAIL"); + } + + /** + * Get the API credentials for GitHub. + * + * @return the API credentials for GitHub (default unset). + */ + public static Optional apiCredentialsGithub() { + return getOptionalEnv("API_CREDENTIALS_GITHUB"); + } + + /** + * Get the IO key for libraries.io. + * + * @return the IO key for libraries.io (default unset). + */ + public static Optional librariesIoKey() { + return getOptionalEnv("LIBRARIES_IO_ACCESS_TOKEN"); + } + + public static boolean isNassaScraperEnabled() { + return "true".equals(System.getenv("ENABLE_NASSA_SCRAPER")); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/InvalidInputException.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/InvalidInputException.java new file mode 100644 index 0000000..9e0971f --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/InvalidInputException.java @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper; + +public class InvalidInputException extends Exception { + + public InvalidInputException(String message) { + super(message); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdRateLimitException.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdRateLimitException.java new file mode 100644 index 0000000..581bcfb --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdRateLimitException.java @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: 2022 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2022 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2023 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper; + +import java.net.URI; + +public class RsdRateLimitException extends RuntimeException { + + public final int statusCode; + public final URI uri; + public final String body; + + public RsdRateLimitException(int statusCode, URI uri, String body, String message) { + super(message); + this.statusCode = statusCode; + this.uri = uri; + this.body = body; + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdResponseException.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdResponseException.java new file mode 100644 index 0000000..cac9317 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdResponseException.java @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2022 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper; + +import java.net.URI; + +public class RsdResponseException extends Exception { + + private static final long serialVersionUID = 1L; + + public final int statusCode; + public final URI uri; + public final String body; + + public RsdResponseException(int statusCode, URI uri, String body, String message) { + super(message); + this.statusCode = statusCode; + this.uri = uri; + this.body = body; + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Utils.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Utils.java new file mode 100644 index 0000000..23c949c --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Utils.java @@ -0,0 +1,371 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +// SPDX-FileCopyrightText: 2022 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2022 dv4all +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper; + +import com.auth0.jwt.JWT; +import com.auth0.jwt.algorithms.Algorithm; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Base64; +import java.util.Date; +import java.util.UUID; +import java.util.function.Function; + +public class Utils { + + private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class); + + // Default timeout used for http connections. + private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(30); + + private Utils() { + } + + /** + * Base64encode a string. + * + * @param s The string to be encoded + * @return The base64 encoded string. + */ + public static String base64Encode(String s) { + return Base64.getEncoder().encodeToString(s.getBytes()); + } + + /** + * Urlencode a string. + * + * @param value The string to be encoded + * @return The urlencoded string. + */ + public static String urlEncode(String value) { + return URLEncoder.encode(value, StandardCharsets.UTF_8); + } + + /** + * Performs a GET request with given headers and returns the response body as a String. + * + * @param uri The encoded URI + * @param headers (Optional) Variable amount of headers. Number of arguments must be a multiple of two. + * @return The response as a String. + * @throws IOException + * @throws InterruptedException + * @throws RsdResponseException + */ + public static String get(String uri, String... headers) throws IOException, InterruptedException, RsdResponseException { + HttpResponse response = getAsHttpResponse(uri, headers); + + if (response.statusCode() >= 300) { + throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response"); + } + + return response.body(); + } + + /** + * Performs a GET request with given headers and returns the entire http response. + * + * @param uri The encoded URI + * @param headers (Optional) Variable amount of headers. Number of arguments must be a multiple of two. + * @return The response as a String. + * @throws IOException + * @throws InterruptedException + */ + public static HttpResponse getAsHttpResponse(String uri, String... headers) throws IOException, InterruptedException { + HttpRequest.Builder httpRequestBuilder = HttpRequest.newBuilder() + .GET() + .timeout(DEFAULT_TIMEOUT) + .uri(URI.create(uri)); + if (headers != null && headers.length > 0 && headers.length % 2 == 0) { + httpRequestBuilder.headers(headers); + } + HttpRequest request = httpRequestBuilder.build(); + + try (HttpClient client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build()) { + return client.send(request, HttpResponse.BodyHandlers.ofString()); + } + } + + /** + * Retrieve data from PostgREST as an admin user and retrieve the response body. + * + * @param uri The URI + * @return Returns the content of the HTTP response + */ + public static String getAsAdmin(String uri) { + String jwtString = adminJwt(); + HttpRequest request = HttpRequest.newBuilder() + .GET() + .uri(URI.create(uri)) + .timeout(DEFAULT_TIMEOUT) + .header("Authorization", "Bearer " + jwtString) + .build(); + + HttpResponse response; + + try (HttpClient client = HttpClient.newHttpClient()) { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (InterruptedException e) { + LOGGER.warn("Request to {} was interrupted", uri, e); + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } catch (IOException e) { + LOGGER.error("An error occurred sending a request to {}", uri, e); + throw new RuntimeException(e); + } + + if (response.statusCode() >= 300) { + throw new RuntimeException("Error fetching data from endpoint " + uri + " with response: " + response.body()); + } + return response.body(); + } + + /** + * Performs a POST request with given headers and returns the response body + * as a String. + * + * @param uri The URI + * @param body the request body as a string + * @param extraHeaders Additional headers (amount must be multiple of two) + * @return The response body as a string + */ + public static String post(String uri, String body, String... extraHeaders) { + HttpRequest.Builder httpRequestBuilder = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(body)) + .timeout(DEFAULT_TIMEOUT) + .uri(URI.create(uri)); + if (extraHeaders != null && extraHeaders.length > 0 && extraHeaders.length % 2 == 0) { + httpRequestBuilder.headers(extraHeaders); + } + HttpRequest request = httpRequestBuilder.build(); + HttpResponse response; + + try (HttpClient client = HttpClient.newHttpClient()) { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + + if (response.statusCode() >= 300) { + throw new RuntimeException("Error fetching data from endpoint " + uri + " with response: " + response.body()); + } + return response.body(); + } + + /** + * Post data to the database. + * + * @param uri The URI + * @param json JSON as a string containing the values + * @param extraHeaders Additional headers (amount must be multiple of two) + * @return ??? + */ + public static String postAsAdmin(String uri, String json, String... extraHeaders) { + String jwtString = adminJwt(); + HttpRequest.Builder builder = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(json)) + .uri(URI.create(uri)) + .timeout(DEFAULT_TIMEOUT) + .header("Content-Type", "application/json") + .header("Authorization", "Bearer " + jwtString); + if (extraHeaders != null && extraHeaders.length > 0) { + builder.headers(extraHeaders); + } + HttpRequest request = builder.build(); + HttpResponse response; + + try (HttpClient client = HttpClient.newHttpClient()) { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + + if (response.statusCode() >= 300) { + throw new RuntimeException("Error fetching data from endpoint " + uri + " with response: " + response.body()); + } + return response.body(); + } + + private static JsonObject basicData(String serviceName, String tableName, UUID referenceId, Exception e) { + String message = e.getMessage(); + StringWriter stringWriter = new StringWriter(); + PrintWriter printWriter = new PrintWriter(stringWriter); + e.printStackTrace(printWriter); + printWriter.flush(); + String stackTrace = stringWriter.toString(); + + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("service_name", serviceName); + jsonObject.addProperty("table_name", tableName); + jsonObject.addProperty("reference_id", referenceId != null ? referenceId.toString() : null); + jsonObject.addProperty("message", message); + jsonObject.addProperty("stack_trace", stackTrace); + + return jsonObject; + } + + public static void saveExceptionInDatabase(String serviceName, String tableName, UUID referenceId, Exception e) { + JsonObject logData = basicData(serviceName, tableName, referenceId, e); + + postAsAdmin(Config.backendBaseUrl() + "/backend_log", logData.toString()); + } + + public static void saveExceptionInDatabase(String serviceName, String tableName, UUID referenceId, RsdResponseException e) { + JsonObject logData = basicData(serviceName, tableName, referenceId, e); + + JsonObject other = new JsonObject(); + other.addProperty("status_code", e.statusCode); + other.addProperty("URI", e.uri != null ? e.uri.toString() : null); + other.addProperty("response_body", e.body); + + logData.add("other_data", other); + + postAsAdmin(Config.backendBaseUrl() + "/backend_log", logData.toString()); + } + + public static void saveExceptionInDatabase(String serviceName, String tableName, UUID referenceId, RsdRateLimitException e) { + JsonObject logData = basicData(serviceName, tableName, referenceId, e); + + JsonObject other = new JsonObject(); + other.addProperty("status_code", e.statusCode); + other.addProperty("URI", e.uri != null ? e.uri.toString() : null); + other.addProperty("response_body", e.body); + + logData.add("other_data", other); + + postAsAdmin(Config.backendBaseUrl() + "/backend_log", logData.toString()); + } + + public static void saveErrorMessageInDatabase(String message, String tableName, String columnName, String primaryKey, String primaryKeyName, ZonedDateTime scrapedAt, String scrapedAtName) { + JsonObject body = new JsonObject(); + if (columnName != null) { + body.addProperty(columnName, message); + } + + if (scrapedAt != null && scrapedAtName != null) { + body.addProperty(scrapedAtName, scrapedAt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + } + + String uri = createPatchUri(Config.backendBaseUrl(), tableName, primaryKey, primaryKeyName); + + patchAsAdmin(uri, body.toString()); + } + + static String createPatchUri(String baseuri, String tableName, String primaryKey, String primaryKeyName) { + return "%s/%s?%s=eq.%s".formatted(baseuri, tableName, primaryKeyName, primaryKey); + } + + public static String patchAsAdmin(String uri, String json, String... extraHeaders) { + String jwtString = adminJwt(); + HttpRequest.Builder builder = HttpRequest.newBuilder() + .method("PATCH", HttpRequest.BodyPublishers.ofString(json)) + .uri(URI.create(uri)) + .timeout(Duration.ofSeconds(30)) + .header("Content-Type", "application/json") + .header("Authorization", "Bearer " + jwtString); + if (extraHeaders != null && extraHeaders.length > 0) { + builder.headers(extraHeaders); + } + HttpRequest request = builder.build(); + HttpResponse response; + try (HttpClient client = HttpClient.newHttpClient()) { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + if (response.statusCode() >= 300) { + throw new RuntimeException("Error fetching data from endpoint " + uri + " with response: " + response.body()); + } + return response.body(); + } + + public static String deleteAsAdmin(String uri, String... extraHeaders) throws IOException, InterruptedException, RsdResponseException { + String jwtString = adminJwt(); + HttpRequest.Builder builder = HttpRequest.newBuilder() + .method("DELETE", HttpRequest.BodyPublishers.noBody()) + .uri(URI.create(uri)) + .timeout(Duration.ofSeconds(30)) + .header("Authorization", "Bearer " + jwtString); + if (extraHeaders != null && extraHeaders.length > 0) { + builder.headers(extraHeaders); + } + HttpRequest request = builder.build(); + HttpResponse response; + try (HttpClient client = HttpClient.newHttpClient()) { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } + if (response.statusCode() >= 300) { + throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Error deleting data as admin"); + } + return response.body(); + } + + private static String adminJwt() { + String signingSecret = Config.jwtSigningSecret(); + Algorithm signingAlgorithm = Algorithm.HMAC256(signingSecret); + return JWT.create() + .withClaim("role", "rsd_admin") + .withExpiresAt(new Date(System.currentTimeMillis() + Config.jwtExpirationTime())) + .sign(signingAlgorithm); + } + + public static String stringOrNull(JsonElement e) { + return e == null || !e.isJsonPrimitive() ? null : e.getAsString(); + } + + public static T safelyGetOrNull(JsonElement element, Function extractor) { + try { + return extractor.apply(element); + } catch (RuntimeException e) { + LOGGER.warn("Exception extracting data from JSON: " + element, e); + return null; + } + } + + public static Integer integerOrNull(JsonElement e) { + return e == null || !e.isJsonPrimitive() ? null : e.getAsInt(); + } + + /** + * Generate a filter to select column entries older than one hour. + * + * @param scrapedAtColumnName the name of the column containing the timestamp. + * @return the filter. + */ + public static String atLeastOneHourAgoFilter(String scrapedAtColumnName) { + String oneHourAgoEncoded = urlEncode(ZonedDateTime.now() + .minusHours(1) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + return "or=(%s.is.null,%s.lte.%s)".formatted(scrapedAtColumnName, scrapedAtColumnName, oneHourAgoEncoded); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/MainAggregator.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/MainAggregator.java new file mode 100644 index 0000000..65735f2 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/MainAggregator.java @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.aggregator; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +public class MainAggregator { + + private static final Logger LOGGER = LoggerFactory.getLogger(MainAggregator.class); + private static final URI BASE_URL = URI.create(Config.backendBaseUrl()); + static final String REMOTE_SOFTWARE_TABLE_NAME = "remote_software"; + static final String AGGREGATOR_SERVICE_NAME = "Aggregator"; + + + public static void main(String[] args) { + LOGGER.info("Start aggregating RSDs"); + long start = System.nanoTime(); + + ZonedDateTime now = ZonedDateTime.now(); + Collection allRemoteEntries = PostgrestConnector.allActiveDomains(BASE_URL); + + Collection remoteEntriesToScrape = allRemoteEntries.stream() + .filter(entry -> entry.refreshedAt() == null || entry.refreshedAt() + .plus(entry.refreshInterval()) + // subtracting 10 seconds to take into account variations in when this scraper starts + .minus(Duration.ofSeconds(10L)) + .isBefore(now)) + .toList(); + + ConcurrentMap softwarePerId = new ConcurrentHashMap<>(remoteEntriesToScrape.size()); + Collection> tasks = remoteEntriesToScrape.stream() + .>map(entry -> () -> { + JsonArray scrapedSoftware = RemoteRsdConnector.getAllSoftware(entry.domain()); + softwarePerId.put(entry.id(), scrapedSoftware); + return null; + }) + .toList(); + + try (ExecutorService executorService = Executors.newFixedThreadPool(8)) { + List> completedFutures = executorService.invokeAll(tasks); + for (Future completedFuture : completedFutures) { + try { + completedFuture.get(); + } catch (ExecutionException e) { + LOGGER.error("Unknown error", e); + Utils.saveExceptionInDatabase(AGGREGATOR_SERVICE_NAME, REMOTE_SOFTWARE_TABLE_NAME, null, e); + } catch (InterruptedException e) { + Utils.saveExceptionInDatabase(AGGREGATOR_SERVICE_NAME, REMOTE_SOFTWARE_TABLE_NAME, null, e); + LOGGER.error("Got interrupted, early exiting aggregating RSDs", e); + Thread.currentThread().interrupt(); + return; + } + } + } catch (InterruptedException e) { + Utils.saveExceptionInDatabase(AGGREGATOR_SERVICE_NAME, REMOTE_SOFTWARE_TABLE_NAME, null, e); + LOGGER.error("Got interrupted, early exiting aggregating RSDs", e); + Thread.currentThread().interrupt(); + return; + } + + JsonArray allSoftware = new JsonArray(); + for (Map.Entry entry : softwarePerId.entrySet()) { + JsonArray softwareArray = entry.getValue(); + UUID id = entry.getKey(); + for (JsonElement jsonElement : softwareArray) { + JsonObject jsonObject = jsonElement.getAsJsonObject(); + jsonObject.addProperty("remote_rsd_id", id.toString()); + jsonObject.addProperty("remote_software_id", jsonObject.getAsJsonPrimitive("id").getAsString()); + jsonObject.remove("id"); + jsonObject + .addProperty("scraped_at", now.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + + allSoftware.add(jsonElement); + } + } + + PostgrestConnector.saveRemoteSoftware(BASE_URL, allSoftware); + + for (UUID id : softwarePerId.keySet()) { + PostgrestConnector.updateRefreshedTimeAndErrorMessage(BASE_URL, id, now, null); + } + + Map> softwareScrapedPerRemoteId = new HashMap<>(); + for (JsonElement jsonElement : allSoftware) { + JsonObject jsonObject = jsonElement.getAsJsonObject(); + UUID remoteRsdId = UUID.fromString(jsonObject.getAsJsonPrimitive("remote_rsd_id").getAsString()); + UUID remoteSoftwareId = UUID.fromString(jsonObject.getAsJsonPrimitive("remote_software_id").getAsString()); + Collection softwareOfRemote = softwareScrapedPerRemoteId.computeIfAbsent(remoteRsdId, id -> new HashSet<>()); + softwareOfRemote.add(remoteSoftwareId); + } + + for (RemoteRsdData remoteRsdData : remoteEntriesToScrape) { + UUID remoteId = remoteRsdData.id(); + if (!softwareScrapedPerRemoteId.containsKey(remoteId)) { + continue; + } + + Collection scrapedSoftware = softwareScrapedPerRemoteId.get(remoteId); + Collection previouslyStoredSoftwareIds = remoteRsdData.softwareIds(); + + for (UUID previouslyStoredSoftwareId : previouslyStoredSoftwareIds) { + if (!scrapedSoftware.contains(previouslyStoredSoftwareId)) { + try { + PostgrestConnector.deleteSoftware(BASE_URL, remoteId, previouslyStoredSoftwareId); + } catch (RsdResponseException e) { + LOGGER.error("Unknown error", e); + Utils.saveExceptionInDatabase(AGGREGATOR_SERVICE_NAME, REMOTE_SOFTWARE_TABLE_NAME, previouslyStoredSoftwareId, e); + } catch (IOException e) { + LOGGER.error("Unknown error", e); + Utils.saveExceptionInDatabase(AGGREGATOR_SERVICE_NAME, REMOTE_SOFTWARE_TABLE_NAME, previouslyStoredSoftwareId, e); + } catch (InterruptedException e) { + LOGGER.error("Got interrupted, early exiting deleting old entries", e); + Utils.saveExceptionInDatabase(AGGREGATOR_SERVICE_NAME, REMOTE_SOFTWARE_TABLE_NAME, previouslyStoredSoftwareId, e); + Thread.currentThread().interrupt(); + return; + } + } + } + } + + for (RemoteRsdData entry : remoteEntriesToScrape) { + UUID id = entry.id(); + if (!softwarePerId.containsKey(id)) { + PostgrestConnector.updateRefreshedTimeAndErrorMessage(BASE_URL, id, now, "Unknown error while scraping"); + } + } + + long stop = System.nanoTime(); + LOGGER.info("Done aggregating RSDs ({} ms)", (stop - start) / 1000_000L); + } + +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/PostgrestConnector.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/PostgrestConnector.java new file mode 100644 index 0000000..f068d10 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/PostgrestConnector.java @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.aggregator; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.UUID; + +public class PostgrestConnector { + + private static final Logger LOGGER = LoggerFactory.getLogger(PostgrestConnector.class); + + private PostgrestConnector() { + } + + public static Collection allActiveDomains(URI baseUrl) { + String filter = "select=id,domain,scrape_interval_minutes,scraped_at,remote_software(remote_software_id)&active=is.true"; + String url = baseUrl.toString() + "/remote_rsd?" + filter; + + String response = Utils.getAsAdmin(url); + + return parseDomainsToScrapeResponse(response); + } + + static List parseDomainsToScrapeResponse(String response) { + JsonArray jsonTree = JsonParser.parseString(response).getAsJsonArray(); + List result = new ArrayList<>(jsonTree.size()); + + for (JsonElement element : jsonTree) { + try { + JsonObject jsonObject = element.getAsJsonObject(); + UUID id = UUID.fromString(jsonObject.getAsJsonPrimitive("id").getAsString()); + URI domain = URI.create(jsonObject.getAsJsonPrimitive("domain").getAsString()); + Duration refreshInterval = Duration.ofMinutes(jsonObject.getAsJsonPrimitive("scrape_interval_minutes") + .getAsLong()); + JsonElement refreshedAtElement = jsonObject.get("scraped_at"); + ZonedDateTime refreshedAt = refreshedAtElement.isJsonNull() ? null : ZonedDateTime.parse(refreshedAtElement.getAsString()); + + Collection softwareIds = new HashSet<>(); + JsonArray idsArray = jsonObject.getAsJsonArray("remote_software"); + for (JsonElement jsonElement : idsArray) { + UUID softwareId = UUID.fromString(jsonElement.getAsJsonObject() + .getAsJsonPrimitive("remote_software_id") + .getAsString()); + softwareIds.add(softwareId); + } + + result.add(new RemoteRsdData( + id, + domain, + refreshInterval, + refreshedAt, + softwareIds + )); + } catch (RuntimeException e) { + LOGGER.error("Exception when parsing item", e); + Utils.saveExceptionInDatabase(MainAggregator.AGGREGATOR_SERVICE_NAME, MainAggregator.REMOTE_SOFTWARE_TABLE_NAME, null, e); + } + } + + return result; + } + + public static void deleteSoftware(URI baseUrl, UUID remoteRsdId, UUID remoteSoftwareId) throws RsdResponseException, IOException, InterruptedException { + String filter = "remote_rsd_id=eq." + remoteRsdId + "&remote_software_id=eq." + remoteSoftwareId; + String url = baseUrl.toString() + "/remote_software?" + filter; + + Utils.deleteAsAdmin(url); + } + + public static void saveRemoteSoftware(URI baseUrl, JsonArray softwareArray) { + String url = baseUrl + "/remote_software?on_conflict=remote_rsd_id,remote_software_id"; + + Utils.postAsAdmin(url, softwareArray.toString(), "Prefer", "resolution=merge-duplicates"); + } + + public static void updateRefreshedTimeAndErrorMessage(URI baseUrl, UUID id, ZonedDateTime refreshedAt, String errorMessage) { + String filter = "id=eq." + id; + String url = baseUrl.toString() + "/remote_rsd?" + filter; + + JsonObject body = new JsonObject(); + body.addProperty("scraped_at", refreshedAt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + body.addProperty("last_err_msg", errorMessage); + + Utils.patchAsAdmin(url, body.toString()); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdConnector.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdConnector.java new file mode 100644 index 0000000..bbdfaf9 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdConnector.java @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.aggregator; + +import com.google.gson.JsonArray; +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; + +import java.io.IOException; +import java.net.URI; +import java.util.StringJoiner; + +public class RemoteRsdConnector { + + private static final String SELECT_LIST; + + static { + // column `categories` from `software_overview()` is not in table `remote_software` + StringJoiner selectListBuilder = new StringJoiner(","); + selectListBuilder.add("id"); + selectListBuilder.add("slug"); + selectListBuilder.add("brand_name"); + selectListBuilder.add("short_statement"); + selectListBuilder.add("image_id"); + selectListBuilder.add("updated_at"); + selectListBuilder.add("contributor_cnt"); + selectListBuilder.add("mention_cnt"); + selectListBuilder.add("is_published"); + selectListBuilder.add("keywords"); + selectListBuilder.add("keywords_text"); + selectListBuilder.add("prog_lang"); + selectListBuilder.add("licenses"); + + SELECT_LIST = selectListBuilder.toString(); + } + + private RemoteRsdConnector() { + } + + public static JsonArray getAllSoftware(URI remoteDomain) throws RsdResponseException, IOException, InterruptedException { + String path = "/api/v1/rpc/software_overview?select=" + SELECT_LIST; + String url = remoteDomain.toString() + path; + + String response = Utils.get(url); + return JsonParser.parseString(response).getAsJsonArray(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdData.java new file mode 100644 index 0000000..fc3a1f0 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdData.java @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.aggregator; + +import java.net.URI; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.UUID; + +public record RemoteRsdData( + UUID id, + URI domain, + Duration refreshInterval, + ZonedDateTime refreshedAt, + Collection softwareIds +) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CitationData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CitationData.java new file mode 100644 index 0000000..3af97d4 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CitationData.java @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import java.util.Collection; +import java.util.UUID; + +/** + * Container class for Citation information retrieved from the database. + */ +public record CitationData( + // UUID of this entry in the database + UUID id, + + // DOI of this entry. + Doi doi, + + // OpenAlex ID of this entry + OpenalexId openalexId, + + // List of known DOIs citing this entry. + Collection knownDois +) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CrossrefMention.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CrossrefMention.java new file mode 100644 index 0000000..67e0a51 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CrossrefMention.java @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +// SPDX-FileCopyrightText: 2022 dv4all +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; + +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +public class CrossrefMention { + + static final Map crossrefTypeMap; + + static { + // https://api.crossref.org/types + crossrefTypeMap = new HashMap<>(); + + crossrefTypeMap.put("book-section", MentionType.bookSection); + crossrefTypeMap.put("monograph", MentionType.other); + crossrefTypeMap.put("report", MentionType.report); + crossrefTypeMap.put("peer-review", MentionType.other); + crossrefTypeMap.put("book-track", MentionType.book); + crossrefTypeMap.put("journal-article", MentionType.journalArticle); + crossrefTypeMap.put("book-part", MentionType.bookSection); + crossrefTypeMap.put("other", MentionType.other); + crossrefTypeMap.put("book", MentionType.book); + crossrefTypeMap.put("journal-volume", MentionType.journalArticle); + crossrefTypeMap.put("book-set", MentionType.book); + crossrefTypeMap.put("reference-entry", MentionType.other); + crossrefTypeMap.put("proceedings-article", MentionType.conferencePaper); + crossrefTypeMap.put("journal", MentionType.journalArticle); + crossrefTypeMap.put("component", MentionType.other); + crossrefTypeMap.put("book-chapter", MentionType.bookSection); + crossrefTypeMap.put("proceedings-series", MentionType.conferencePaper); + crossrefTypeMap.put("report-series", MentionType.report); + crossrefTypeMap.put("proceedings", MentionType.conferencePaper); + crossrefTypeMap.put("standard", MentionType.other); + crossrefTypeMap.put("reference-book", MentionType.book); + crossrefTypeMap.put("posted-content", MentionType.other); + crossrefTypeMap.put("journal-issue", MentionType.journalArticle); + crossrefTypeMap.put("dissertation", MentionType.thesis); + crossrefTypeMap.put("grant", MentionType.other); + crossrefTypeMap.put("dataset", MentionType.dataset); + crossrefTypeMap.put("book-series", MentionType.book); + crossrefTypeMap.put("edited-book", MentionType.book); + crossrefTypeMap.put("standard-series", MentionType.other); + } + + private final Doi doi; + + public CrossrefMention(Doi doi) { + this.doi = Objects.requireNonNull(doi); + } + + public ExternalMentionRecord mentionData() throws IOException, InterruptedException, RsdResponseException { + StringBuilder crossrefUrlBuilder = new StringBuilder("https://api.crossref.org/works/" + Utils.urlEncode(doi.toString())); + Config.crossrefContactEmail().ifPresent(email -> crossrefUrlBuilder.append("?mailto=").append(email)); + String responseJson = Utils.get(crossrefUrlBuilder.toString()); + JsonObject jsonTree = JsonParser.parseString(responseJson).getAsJsonObject(); + JsonObject workJson = jsonTree.getAsJsonObject("message"); + + URI mentionUrl = URI.create("https://doi.org/" + Utils.urlEncode(this.doi.toString())); + String title = workJson.getAsJsonArray("title").get(0).getAsString(); + + Collection authorsBuilder = new ArrayList<>(); + String authors = null; + Iterable authorsJson = (Iterable) workJson.getAsJsonArray("author"); + if (authorsJson != null) { + for (JsonObject authorJson : authorsJson) { + String givenName = Utils.stringOrNull(authorJson.get("given")); + String familyName = Utils.stringOrNull(authorJson.get("family")); + String name = Utils.stringOrNull(authorJson.get("name")); + if (givenName != null && familyName != null) { + authorsBuilder.add(givenName + " " + familyName); + } else if (name != null) { + authorsBuilder.add(name); + } else if (givenName != null) { + authorsBuilder.add(givenName); + } else if (familyName != null) { + authorsBuilder.add(familyName); + } + } + authors = String.join(", ", authorsBuilder); + } + + String publisher = Utils.stringOrNull(workJson.get("publisher")); + Integer publicationYear = null; + try { + publicationYear = Utils.integerOrNull(workJson.getAsJsonObject("published").getAsJsonArray("date-parts").get(0).getAsJsonArray().get(0)); + } catch (RuntimeException e) { + // year not found, we leave it at null, nothing to do + } + String journal = null; + if (!workJson.getAsJsonArray("container-title").isEmpty()) { + JsonArray journalTitles = workJson.getAsJsonArray("container-title"); + StringBuilder journalBuilder = new StringBuilder(journalTitles.get(0).getAsString()); + for (int i = 1; i < journalTitles.size(); i++) { + journalBuilder.append(", ").append(journalTitles.get(i).getAsString()); + } + journal = journalBuilder.toString(); + } + String page = Utils.stringOrNull(workJson.get("page")); + MentionType mentionType = crossrefTypeMap.getOrDefault(Utils.stringOrNull(workJson.get("type")), MentionType.other); + + return new ExternalMentionRecord( + this.doi, + null, + null, + mentionUrl, + title, + authors, + publisher, + publicationYear, + journal, + page, + mentionType, + "Crossref", + null + ); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataCiteReleaseRepository.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataCiteReleaseRepository.java new file mode 100644 index 0000000..f210572 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataCiteReleaseRepository.java @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class DataCiteReleaseRepository { + + private static final Logger LOGGER = LoggerFactory.getLogger(DataCiteReleaseRepository.class); + + // editorconfig-checker-disable + private static final String QUERY_UNFORMATTED = """ + query { + works(ids: [%s], first: 10000) { + nodes { + doi + versionOfCount + relatedIdentifiers { + relationType + relatedIdentifierType + relatedIdentifier + } + } + } + } + """; + // editorconfig-checker-enable + + public Map> getVersionedDois(Collection conceptDois) { + if (conceptDois.isEmpty()) { + return Collections.emptyMap(); + } + + String query = QUERY_UNFORMATTED.formatted(DataciteMentionRepository.joinDoisForGraphqlQuery(conceptDois)); + JsonObject body = new JsonObject(); + body.addProperty("query", query); + String responseJson = Utils.post("https://api.datacite.org/graphql", body.toString(), "Content-Type", "application/json"); + return parseJson(responseJson); + } + + Map> parseJson(String json) { + DataciteMentionRepository dataciteMentionRepository = new DataciteMentionRepository(); + + JsonObject root = JsonParser.parseString(json).getAsJsonObject(); + JsonArray worksJson = root.getAsJsonObject("data").getAsJsonObject("works").getAsJsonArray("nodes"); + Map> releasesPerConceptDoi = new HashMap<>(); + for (JsonElement work : worksJson) { + try { + JsonObject workObject = work.getAsJsonObject(); + String conceptDoiString = workObject.getAsJsonPrimitive("doi").getAsString(); + Doi conceptDoi = Doi.fromString(conceptDoiString); + Integer versionOfCount = Utils.integerOrNull(workObject.get("versionOfCount")); + if (versionOfCount == null || versionOfCount.intValue() != 0) { + LOGGER.debug("{} is not a concept DOI", conceptDoiString); + continue; + } + + Collection versionDois = new ArrayList<>(); + JsonArray relatedIdentifiers = workObject.getAsJsonArray("relatedIdentifiers"); + for (JsonElement relatedIdentifier : relatedIdentifiers) { + JsonObject relatedIdentifierObject = relatedIdentifier.getAsJsonObject(); + String relationType = relatedIdentifierObject.getAsJsonPrimitive("relationType").getAsString(); + if (relationType == null || !relationType.equals("HasVersion")) continue; + + String relatedIdentifierType = relatedIdentifierObject.getAsJsonPrimitive("relatedIdentifierType").getAsString(); + if (relatedIdentifierType == null || !relatedIdentifierType.equals("DOI")) continue; + + String relatedIdentifierDoi = relatedIdentifierObject.getAsJsonPrimitive("relatedIdentifier").getAsString(); + versionDois.add(Doi.fromString(relatedIdentifierDoi)); + } + Collection versionedMentions = dataciteMentionRepository.mentionData(versionDois); + + releasesPerConceptDoi.put(conceptDoi, versionedMentions); + } catch (RuntimeException e) { + LOGGER.error("Failed to scrape a DataCite mention with data {}, ", work, e); + } + } + return releasesPerConceptDoi; + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java new file mode 100644 index 0000000..d4d048a --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java @@ -0,0 +1,229 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class DataciteMentionRepository { + + private static final Logger LOGGER = LoggerFactory.getLogger(DataciteMentionRepository.class); + + // editorconfig-checker-disable + private static final String QUERY_UNFORMATTED = """ + query { + works(ids: [%s], first: 10000) { + nodes { + doi + types { + resourceType + resourceTypeGeneral + } + version + relatedIdentifiers { + relatedIdentifier + relatedIdentifierType + } + titles(first: 1) { + title + } + publisher { + name + } + publicationYear + registered + creators { + givenName + familyName + } + contributors { + givenName + familyName + } + } + } + }"""; + // editorconfig-checker-enable + + private static final Map dataciteTypeMap; + private static final Map dataciteTextTypeMap; + private static final Pattern URL_TREE_TAG_PATTERN = Pattern.compile("/tree/([^/]+)$"); + + static { + // https://schema.datacite.org/meta/kernel-4.4/ + dataciteTypeMap = new HashMap<>(); + dataciteTypeMap.put("Audiovisual", MentionType.presentation); + dataciteTypeMap.put("Book", MentionType.book); + dataciteTypeMap.put("BookChapter", MentionType.bookSection); + dataciteTypeMap.put("Collection", MentionType.other); + dataciteTypeMap.put("ComputationalNotebook", MentionType.computerProgram); + dataciteTypeMap.put("ConferencePaper", MentionType.conferencePaper); + dataciteTypeMap.put("ConferenceProceeding", MentionType.conferencePaper); + dataciteTypeMap.put("DataPaper", MentionType.other); + dataciteTypeMap.put("Dataset", MentionType.dataset); + dataciteTypeMap.put("Dissertation", MentionType.thesis); + dataciteTypeMap.put("Event", MentionType.workshop); + dataciteTypeMap.put("Image", MentionType.other); + dataciteTypeMap.put("InteractiveResource", MentionType.other); + dataciteTypeMap.put("Journal", MentionType.journalArticle); + dataciteTypeMap.put("JournalArticle", MentionType.journalArticle); + dataciteTypeMap.put("Model", MentionType.other); + dataciteTypeMap.put("OutputManagementPlan", MentionType.other); + dataciteTypeMap.put("PeerReview", MentionType.other); + dataciteTypeMap.put("Preprint", MentionType.other); + dataciteTypeMap.put("PhysicalObject", MentionType.other); + dataciteTypeMap.put("Report", MentionType.report); + dataciteTypeMap.put("Service", MentionType.other); + dataciteTypeMap.put("Software", MentionType.computerProgram); + dataciteTypeMap.put("Sound", MentionType.other); + dataciteTypeMap.put("Standard", MentionType.other); + // dataciteTypeMap.put("Text", MentionType.other); + dataciteTypeMap.put("Workflow", MentionType.other); + dataciteTypeMap.put("Other", MentionType.other); + + dataciteTextTypeMap = new HashMap<>(); + dataciteTextTypeMap.put("Conference paper", MentionType.conferencePaper); + dataciteTextTypeMap.put("Dissertation", MentionType.thesis); + dataciteTextTypeMap.put("Journal article", MentionType.journalArticle); + dataciteTextTypeMap.put("Poster", MentionType.poster); + dataciteTextTypeMap.put("Presentation", MentionType.presentation); + dataciteTextTypeMap.put("Report", MentionType.report); + } + + // "10.5281/zenodo.1408128","10.1186/s12859-018-2165-7" + static String joinDoisForGraphqlQuery(Collection dois) { + return dois.stream() + .map(Doi::toString) + .collect(Collectors.joining("\",\"", "\"", "\"")); + } + + static Collection jsonStringToUniqueMentions(String json) { + JsonObject root = JsonParser.parseString(json).getAsJsonObject(); + JsonArray worksJson = root.getAsJsonObject("data").getAsJsonObject("works").getAsJsonArray("nodes"); + Collection mentions = new ArrayList<>(); + Set usedDois = new HashSet<>(); + for (JsonElement work : worksJson) { + try { + // Sometimes, DataCite gives back two of the same results for one DOI, e.g. for 10.4122/1.1000000817, + // so we need to only add it once, otherwise we cannot POST it to the backend + ExternalMentionRecord parsedMention = parseWork(work.getAsJsonObject()); + if (usedDois.contains(parsedMention.doi())) continue; + + usedDois.add(parsedMention.doi()); + mentions.add(parsedMention); + } catch (RuntimeException e) { + // TODO: fix exception type + LOGGER.error("Failed to scrape a DataCite mention with data {} ", work, e); + } + } + return mentions; + } + + static ExternalMentionRecord parseWork(JsonObject work) { + Doi doi = Doi.fromString(work.get("doi").getAsString()); + URI url = URI.create("https://doi.org/" + Utils.urlEncode(doi.toString())); + String title = work.getAsJsonArray("titles").get(0).getAsJsonObject().get("title").getAsString(); + + Collection authorsBuilder = new ArrayList<>(); + Iterable creators = (Iterable) work.getAsJsonArray("creators"); + for (JsonObject creator : creators) { + addAuthor(authorsBuilder, creator); + } + Iterable contributors = (Iterable) work.getAsJsonArray("contributors"); + for (JsonObject contributor : contributors) { + addAuthor(authorsBuilder, contributor); + } + String authors = String.join(", ", authorsBuilder); + + String publisher = Utils.stringOrNull(work.getAsJsonObject("publisher").get("name")); + Integer publicationYear = Utils.integerOrNull(work.get("publicationYear")); + String doiRegistrationDateString = Utils.stringOrNull(work.get("registered")); + ZonedDateTime doiRegistrationDate = null; + if (doiRegistrationDateString != null) { + doiRegistrationDate = ZonedDateTime.parse(doiRegistrationDateString); + } + + MentionType mentionType; + String dataciteResourceTypeGeneral = Utils.stringOrNull(work.getAsJsonObject("types").get("resourceTypeGeneral")); + if (dataciteResourceTypeGeneral != null && dataciteResourceTypeGeneral.equals("Text")) { + String dataciteResourceType = Utils.stringOrNull(work.getAsJsonObject("types").get("resourceType")); + if (dataciteResourceType != null) dataciteResourceType = dataciteResourceType.strip(); + mentionType = dataciteTextTypeMap.getOrDefault(dataciteResourceType, MentionType.other); + } else { + mentionType = dataciteTypeMap.getOrDefault(dataciteResourceTypeGeneral, MentionType.other); + } + String version = Utils.stringOrNull(work.get("version")); + // if the version is null, we can often get the version from a linked Git URL which ends in "/tree/{tag}" + if (version == null) { + JsonArray relatedIdentifiers = work.getAsJsonArray("relatedIdentifiers"); + for (JsonElement relatedIdentifier : relatedIdentifiers) { + String relatedIdentifierString = Utils.stringOrNull(relatedIdentifier.getAsJsonObject().get("relatedIdentifier")); + String relatedIdentifierType = Utils.stringOrNull(relatedIdentifier.getAsJsonObject().get("relatedIdentifierType")); + if (relatedIdentifierString != null && relatedIdentifierType != null && relatedIdentifierType.equals("URL")) { + Matcher tagMatcher = URL_TREE_TAG_PATTERN.matcher(relatedIdentifierString); + if (tagMatcher.find()) { + version = tagMatcher.group(1); + break; + } + } + } + } + + return new ExternalMentionRecord( + doi, + doiRegistrationDate, + null, + url, + title, + authors, + publisher, + publicationYear, + null, + null, + mentionType, + "DataCite", + version + ); + } + + static void addAuthor(Collection authors, JsonObject author) { + String givenName = Utils.stringOrNull(author.get("givenName")); + String familyName = Utils.stringOrNull(author.get("familyName")); + if (givenName == null && familyName == null) return; + if (givenName == null) authors.add(familyName); + else if (familyName == null) authors.add(givenName); + else authors.add(givenName + " " + familyName); + } + + public Collection mentionData(Collection dois) { + if (dois == null || dois.isEmpty()) { + return Collections.emptyList(); + } + + JsonObject body = new JsonObject(); + body.addProperty("query", QUERY_UNFORMATTED.formatted(joinDoisForGraphqlQuery(dois))); + String responseJson = Utils.post("https://api.datacite.org/graphql", body.toString(), "Content-Type", "application/json"); + return jsonStringToUniqueMentions(responseJson); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/Doi.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/Doi.java new file mode 100644 index 0000000..034603f --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/Doi.java @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import nl.esciencecenter.rsd.scraper.Utils; + +import java.util.Locale; +import java.util.regex.Pattern; + +public class Doi { + + private static final Pattern DOI_PATTERN = Pattern.compile("^10(\\.\\w+)+/\\S+$"); + + private final String doiString; + + private Doi(String doiString) { + this.doiString = doiString.toLowerCase(Locale.ROOT); + } + + public static boolean isValid(String doiToTest) { + return doiToTest != null && doiToTest.length() <= 255 && DOI_PATTERN.asPredicate().test(doiToTest); + } + + public static Doi fromString(String doi) { + if (isValid(doi)) { + return new Doi(doi); + } else { + throw new IllegalArgumentException(); + } + } + + public String toUrlEncodedString() { + return Utils.urlEncode(doiString); + } + + @Override + public String toString() { + return doiString; + } + + @Override + public int hashCode() { + return doiString.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (this == other) { + return true; + } + if (other instanceof Doi otherDoi) { + return doiString.equals(otherDoi.doiString); + } + + return false; + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ExternalMentionRecord.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ExternalMentionRecord.java new file mode 100644 index 0000000..89973a2 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ExternalMentionRecord.java @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import java.net.URI; +import java.time.ZonedDateTime; + +public record ExternalMentionRecord( + Doi doi, + ZonedDateTime doiRegistrationDate, + OpenalexId openalexId, + URI url, + String title, + String authors, + String publisher, + Integer publicationYear, + String journal, + String page, + MentionType mentionType, + String source, + String version +) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainCitations.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainCitations.java new file mode 100644 index 0000000..da9559c --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainCitations.java @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.UUID; + +/* + * Main entry point for citation scraper. + */ +public class MainCitations { + + private static final Logger LOGGER = LoggerFactory.getLogger(MainCitations.class); + + public static void main(String[] args) { + + LOGGER.info("Start scraping citations"); + + long start = System.currentTimeMillis(); + + try { + // Connect to the database to retrieve the reference papers to scrape + + String backendUrl = Config.backendBaseUrl(); + PostgrestCitationRepository localCitationRepository = new PostgrestCitationRepository(backendUrl); + + Collection referencePapersToScrape = localCitationRepository.leastRecentlyScrapedCitations(5); + OpenAlexConnector openAlexConnector = new OpenAlexConnector(); + PostgrestMentionRepository localMentionRepository = new PostgrestMentionRepository(backendUrl); + String email = Config.crossrefContactEmail().orElse(null); + Instant now = Instant.now(); + + for (CitationData citationData : referencePapersToScrape) { + + long t1 = System.currentTimeMillis(); + + LOGGER.info("Scraping for DOI {}, OpenAlex ID {}", citationData.doi(), citationData.openalexId()); + + Collection citingMentions = openAlexConnector.citations(citationData.openalexId(), citationData.doi(), email, citationData.id()); + // we don't update mentions that have a DOI in the database with OpenAlex data, as they can already be + // scraped through Crossref or DataCite + + long t2 = System.currentTimeMillis(); + + citingMentions.removeIf(mention -> mention.doi() != null && citationData.knownDois().contains(mention.doi())); + Collection savedIds = new ArrayList<>(citingMentions.size()); + for (ExternalMentionRecord citingMention : citingMentions) { + try { + RsdMentionIds ids = localMentionRepository.createOrUpdateMentionWithOpenalexId(citingMention, now); + savedIds.add(ids); + } catch (Exception e) { + LOGGER.error("Unable to save exception with OpenAlex ID {}", citingMention.openalexId()); + Utils.saveExceptionInDatabase("Citation scraper", "mention", null, e); + } + } + + Collection citingMentionIds = new ArrayList<>(); + for (RsdMentionIds ids : savedIds) { + citingMentionIds.add(ids.id()); + } + + long t3 = System.currentTimeMillis(); + + localCitationRepository.saveCitations(backendUrl, citationData.id(), citingMentionIds, now); + + long t4 = System.currentTimeMillis(); + + LOGGER.info("Scraping for {} done. OpenAlex: {} ms. Saving mentions {} ms. Saving citations {} ms. Total {} ms.", citationData.doi(), (t2 - t1), (t3 - t2), (t4 - t3), (t4 - t1)); + } + + } catch (IOException | InterruptedException e) { + Utils.saveExceptionInDatabase("Citation scraper", null, null, e); + + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + } + + long time = System.currentTimeMillis() - start; + LOGGER.info("Done scraping citations ({} ms.)", time); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainMentions.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainMentions.java new file mode 100644 index 0000000..4661e27 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainMentions.java @@ -0,0 +1,211 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +public class MainMentions { + + private static final Logger LOGGER = LoggerFactory.getLogger(MainMentions.class); + + public static void main(String[] args) { + + LOGGER.info("Start scraping mentions"); + + long t1 = System.currentTimeMillis(); + + PostgrestMentionRepository localMentionRepository = new PostgrestMentionRepository(Config.backendBaseUrl()); + Collection mentionsToScrape = localMentionRepository.leastRecentlyScrapedMentions(Config.maxRequestsDoi()); + // we will remove successfully scraped mentions from here, + // we use this to set scrapedAt even for failed mentions, + // to put them back at the scraping queue + Map mentionsFailedToScrape = new HashMap<>(); + Map doiToId = new HashMap<>(); + Map openalexIdToId = new HashMap<>(); + for (RsdMentionIds mentionIds : mentionsToScrape) { + UUID id = mentionIds.id(); + mentionsFailedToScrape.put(id, mentionIds); + + Doi doi = mentionIds.doi(); + if (doi != null) { + doiToId.put(doi, id); + } + + OpenalexId openalexId = mentionIds.openalexId(); + if (openalexId != null) { + openalexIdToId.put(openalexId, id); + } + } + + String doisJoined = mentionsToScrape.stream() + .map(RsdMentionIds::doi) + .filter(Objects::nonNull) + .map(Doi::toUrlEncodedString) + .collect(Collectors.joining(",")); + String jsonSources = null; + try { + jsonSources = Utils.get("https://doi.org/doiRA/" + doisJoined); + } catch (Exception e) { + Utils.saveExceptionInDatabase("DOI mention scraper", "mention", null, e); + System.exit(1); + } + + Map doiToSource = parseJsonDoiSources(jsonSources); + + Instant now = Instant.now(); + + // DATACITE + final String dataciteDoiRaKey = "DataCite"; + Collection dataciteDois = doiToSource.entrySet() + .stream() + .filter(doiSourceEntry -> doiSourceEntry.getValue().equals(dataciteDoiRaKey)) + .map(Map.Entry::getKey) + .map(Doi::fromString) + .toList(); + Collection scrapedDataciteMentions = List.of(); + try { + scrapedDataciteMentions = new DataciteMentionRepository().mentionData(dataciteDois); + } catch (RuntimeException e) { + Exception exceptionToSave = new Exception("Failed scraping the following DataCite DOIs: " + dataciteDois, e); + Utils.saveExceptionInDatabase("DataCite mention scraper", "mention", null, exceptionToSave); + } + for (ExternalMentionRecord scrapedMention : scrapedDataciteMentions) { + Doi doi = scrapedMention.doi(); + UUID id = doiToId.get(doi); + try { + RsdMentionRecord mentionToUpdate = new RsdMentionRecord(id, scrapedMention, now); + localMentionRepository.updateMention(mentionToUpdate, false); + mentionsFailedToScrape.remove(id); + } catch (Exception e) { + LOGGER.error("Failed to update a DataCite mention with DOI {}", scrapedMention.doi()); + Utils.saveExceptionInDatabase("Mention scraper", "mention", id, e); + } + + } + // END DATACITE + + // CROSSREF + final String crossrefDoiRaKey = "Crossref"; + Collection crossrefDois = doiToSource.entrySet() + .stream() + .filter(doiSourceEntry -> doiSourceEntry.getValue().equals(crossrefDoiRaKey)) + .map(Map.Entry::getKey) + .map(Doi::fromString) + .toList(); + for (Doi crossrefDoi : crossrefDois) { + ExternalMentionRecord scrapedMention; + UUID id = doiToId.get(crossrefDoi); + try { + scrapedMention = new CrossrefMention(crossrefDoi).mentionData(); + } catch (Exception e) { + LOGGER.error("Failed to scrape a Crossref mention with DOI {}", crossrefDoi); + RuntimeException exceptionWithMessage = new RuntimeException("Failed to scrape a Crossref mention with DOI " + crossrefDoi, e); + Utils.saveExceptionInDatabase("Crossref mention scraper", "mention", id, exceptionWithMessage); + continue; + } + RsdMentionRecord mentionToUpdate = new RsdMentionRecord(id, scrapedMention, now); + try { + localMentionRepository.updateMention(mentionToUpdate, false); + mentionsFailedToScrape.remove(id); + } catch (Exception e) { + RuntimeException exceptionWithMessage = new RuntimeException("Failed to update a Crossref mention with DOI " + crossrefDoi, e); + Utils.saveExceptionInDatabase("Crossref mention scraper", "mention", id, exceptionWithMessage); + } + } + // END CROSSREF + + // OPENALEX (other DOI registry agents) + String email = Config.crossrefContactEmail().orElse(null); + Collection scrapedOpenalexMentions = new ArrayList<>(); + OpenAlexConnector openAlexConnector = new OpenAlexConnector(); + Collection invalidDoiRas = Set.of(dataciteDoiRaKey, crossrefDoiRaKey, "Invalid DOI", "DOI does not exist", "Unknown"); + Collection europeanPublicationsOfficeDois = doiToSource.entrySet() + .stream() + .filter(doiSourceEntry -> !invalidDoiRas.contains(doiSourceEntry.getValue())) + .map(Map.Entry::getKey) + .map(Doi::fromString) + .toList(); + try { + scrapedOpenalexMentions.addAll(openAlexConnector.mentionDataByDois(europeanPublicationsOfficeDois, email)); + } catch (Exception e) { + Exception exceptionToSave = new Exception("Failed scraping the following EPO DOIs: " + europeanPublicationsOfficeDois, e); + Utils.saveExceptionInDatabase("OpenAlex mention scraper", "mention", null, exceptionToSave); + } + Collection openalexIdsToScrape = mentionsToScrape + .stream() + .filter(ids -> ids.doi() == null && ids.openalexId() != null) + .map(RsdMentionIds::openalexId) + .toList(); + try { + scrapedOpenalexMentions.addAll(openAlexConnector.mentionDataByOpenalexIds(openalexIdsToScrape, email)); + } catch (Exception e) { + Exception exceptionToSave = new Exception("Failed scraping the following OpenAlex IDs: " + openalexIdsToScrape, e); + Utils.saveExceptionInDatabase("OpenAlex mention scraper", "mention", null, exceptionToSave); + } + + for (ExternalMentionRecord scrapedMention : scrapedOpenalexMentions) { + OpenalexId openalexId = scrapedMention.openalexId(); + UUID id = openalexIdToId.get(openalexId); + RsdMentionRecord mentionToUpdate = new RsdMentionRecord(id, scrapedMention, now); + try { + localMentionRepository.updateMention(mentionToUpdate, true); + mentionsFailedToScrape.remove(id); + } catch (Exception e) { + LOGGER.error("Failed to update an OpenAlex mention with DOI {}", scrapedMention.doi()); + Utils.saveExceptionInDatabase("Mention scraper", "mention", id, e); + } + } + // END OPENALEX + + for (RsdMentionIds ids : mentionsFailedToScrape.values()) { + LOGGER.error("Failed to scrape mention with DOI {}", ids.doi()); + try { + localMentionRepository.saveScrapedAt(ids, now); + } catch (RuntimeException e) { + Utils.saveExceptionInDatabase("Mention scraper", "mention", ids.id(), e); + } + } + + long time = System.currentTimeMillis() - t1; + LOGGER.info("Done scraping mentions ({} ms.)", time); + } + + static Map parseJsonDoiSources(String jsonSources) { + JsonArray sourcesArray = JsonParser.parseString(jsonSources).getAsJsonArray(); + Map result = new HashMap<>(); + for (JsonElement jsonElement : sourcesArray) { + JsonObject jsonPair = jsonElement.getAsJsonObject(); + String doi = jsonPair.getAsJsonPrimitive("DOI").getAsString(); + JsonElement jsonStatus = jsonPair.get("status"); + if (jsonStatus == null) { + String source = jsonPair.getAsJsonPrimitive("RA").getAsString(); + result.put(doi, source); + } else { + String status = jsonStatus.getAsString(); + result.put(doi, status); + } + } + return result; + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainReleases.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainReleases.java new file mode 100644 index 0000000..98cb886 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainReleases.java @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +/* + * 1. Get the least recently scraped releases from software with a concept DOI. We also check for existing releases that already exist as a mention in the database, so we don't have to (TODO) recreate them later. + * 2. For each release check if it's a concept DOI on DataCite and get all the versioned DOIs. + * 3. For each versioned DOI, get its metadata from DataCite. If this versioned DOI did not exist as a mention, create it using the mention scraper. Add each versioned DOI as an entry in the release_content table + */ +public class MainReleases { + + private static final Logger LOGGER = LoggerFactory.getLogger(MainReleases.class); + + public static void main(String[] args) { + + LOGGER.info("Start scraping releases"); + + long t1 = System.currentTimeMillis(); + + PostgrestReleaseRepository releaseRepository = new PostgrestReleaseRepository(Config.backendBaseUrl()); + + Collection releasesToScrape = releaseRepository.leastRecentlyScrapedReleases(Config.maxRequestsDoi()); + + Collection conceptDoisToScrape = releasesToScrape.stream() + .map(releaseData -> releaseData.conceptDoi) + .toList(); + + Map> scrapedReleasesPerConceptDoi = new DataCiteReleaseRepository().getVersionedDois(conceptDoisToScrape); + + Instant now = Instant.now(); + PostgrestMentionRepository localMentionRepository = new PostgrestMentionRepository(Config.backendBaseUrl()); + Collection allMentions = scrapedReleasesPerConceptDoi.values().stream() + .flatMap(Collection::stream) + .toList(); + Map doiToId = new HashMap<>(); + for (ExternalMentionRecord mention : allMentions) { + try { + RsdMentionIds ids = localMentionRepository.createMentionIfNotExistsOnDoiAndGetIds(mention, now); + doiToId.put(mention.doi(), ids.id()); + } catch (Exception e) { + LOGGER.error("Unable to save mention with DOI {}", mention.doi()); + Utils.saveExceptionInDatabase("Releases scraper", "mention", null, e); + } + } + + releaseRepository.saveReleaseContent(releasesToScrape, scrapedReleasesPerConceptDoi, doiToId); + + long time = System.currentTimeMillis() - t1; + LOGGER.info("Done scraping releases ({} ms.)", time); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MentionType.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MentionType.java new file mode 100644 index 0000000..793a965 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MentionType.java @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: 2022 - 2023 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2023 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +public enum MentionType { + blogPost, + book, + bookSection, + computerProgram, + conferencePaper, + dataset, + highlight, + interview, + journalArticle, + magazineArticle, + newspaperArticle, + poster, + presentation, + report, + thesis, + videoRecording, + webpage, + workshop, + other +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnector.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnector.java new file mode 100644 index 0000000..7715843 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnector.java @@ -0,0 +1,297 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpResponse; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +class OpenAlexConnector { + + private static final Logger LOGGER = LoggerFactory.getLogger(OpenAlexConnector.class); + + static final String DOI_FILTER_URL_UNFORMATTED = "https://api.openalex.org/works?filter=doi:%s"; + static final String OPENALEX_ID_URL_UNFORMATTED = "https://api.openalex.org/works?filter=ids.openalex:%s"; + + public Collection mentionDataByDois(Collection dois, String email) throws IOException, InterruptedException { + if (dois == null || dois.isEmpty()) { + return Collections.emptyList(); + } + + String filter = dois + .stream() + .filter(Objects::nonNull) + .map(Doi::toString) + .collect(Collectors.joining("|")); + // e.g. https://api.openalex.org/works?filter=doi:10.1038%2Fs41598-024-73248-4|10.5194%2Ftc-2022-249-rc1&per-page=200 + String worksUri = DOI_FILTER_URL_UNFORMATTED.formatted(Utils.urlEncode(filter)) + "&per-page=200"; + + HttpResponse response; + if (email == null || email.isBlank()) { + response = Utils.getAsHttpResponse(worksUri); + } else { + response = Utils.getAsHttpResponse(worksUri, "User-Agent", "mailto:" + email); + } + + JsonObject tree = JsonParser.parseString(response.body()).getAsJsonObject(); + JsonArray mentionsArray = tree + .getAsJsonArray("results"); + + Collection mentions = new ArrayList<>(); + for (JsonElement mention : mentionsArray) { + ExternalMentionRecord citationAsMention; + try { + citationAsMention = parseAsMention(mention); + } catch (RuntimeException e) { + Utils.saveExceptionInDatabase("OpenAlex mention scraper", "mention", null, e); + continue; + } + mentions.add(citationAsMention); + } + + return mentions; + } + + public Collection mentionDataByOpenalexIds(Collection openalexIds, String email) throws IOException, InterruptedException { + if (openalexIds == null || openalexIds.isEmpty()) { + return Collections.emptyList(); + } + + String filter = openalexIds + .stream() + .filter(Objects::nonNull) + .map(OpenalexId::getOpenalexKey) + .collect(Collectors.joining("|")); + // e.g. https://api.openalex.org/works?filter=ids.openalex:W4402994101|W4319593220&per-page=200 + String worksUri = OPENALEX_ID_URL_UNFORMATTED.formatted(Utils.urlEncode(filter)) + "&per-page=200"; + + HttpResponse response; + if (email == null || email.isBlank()) { + response = Utils.getAsHttpResponse(worksUri); + } else { + response = Utils.getAsHttpResponse(worksUri, "User-Agent", "mailto:" + email); + } + + JsonObject tree = JsonParser.parseString(response.body()).getAsJsonObject(); + JsonArray mentionsArray = tree + .getAsJsonArray("results"); + + Collection mentions = new ArrayList<>(); + for (JsonElement mention : mentionsArray) { + ExternalMentionRecord citationAsMention; + try { + citationAsMention = parseAsMention(mention); + } catch (RuntimeException e) { + Utils.saveExceptionInDatabase("OpenAlex mention scraper", "mention", null, e); + continue; + } + mentions.add(citationAsMention); + } + + return mentions; + } + + public Collection citations(OpenalexId openalexId, Doi doi, String email, UUID id) throws IOException, InterruptedException { + // This shouldn't happen, but let's check it to prevent unexpected exceptions: + if (doi == null && openalexId == null) { + return Collections.emptyList(); + } + + String worksUri = openalexId != null + ? OPENALEX_ID_URL_UNFORMATTED.formatted(openalexId.toUrlEncodedString()) + : DOI_FILTER_URL_UNFORMATTED.formatted(doi.toUrlEncodedString()); + + Optional optionalCitationsUri = citationsUri(worksUri, email); + if (optionalCitationsUri.isEmpty()) { + return Collections.emptyList(); + } + + return scrapeCitations(optionalCitationsUri.get(), email, id); + } + + static Optional citationsUri(String worksUri, String email) throws IOException, InterruptedException { + HttpResponse response; + if (email == null || email.isBlank()) { + response = Utils.getAsHttpResponse(worksUri); + } else { + response = Utils.getAsHttpResponse(worksUri, "User-Agent", "mailto:" + email); + } + + JsonObject tree = JsonParser.parseString(response.body()).getAsJsonObject(); + + int count = tree + .getAsJsonObject("meta") + .getAsJsonPrimitive("count") + .getAsInt(); + + if (count < 1) { + LOGGER.warn("No results found for {}: {}", worksUri, count); + return Optional.empty(); + } + + if (count > 1) { + LOGGER.warn("More than 1 result found for {}: {}, taking the first", worksUri, count); + } + + String citationsUri = null; + try { + citationsUri = tree + .getAsJsonArray("results") + .get(0) + .getAsJsonObject() + .getAsJsonPrimitive("cited_by_api_url") + .getAsString(); + } catch (RuntimeException e) { + LOGGER.error("Exception parsing cited_by_api_url for %s".formatted(worksUri), e); + Utils.saveExceptionInDatabase("OpenAlex citations scraper", null, null, e); + } + + return Optional.ofNullable(citationsUri); + } + + // we use cursor paging as that will always work + // https://docs.openalex.org/how-to-use-the-api/get-lists-of-entities/paging#cursor-paging + static Collection scrapeCitations(String citationsUri, String email, UUID id) throws IOException, InterruptedException { + final int perPage = 200; + String cursor = "*"; + + Collection citations = new ArrayList<>(); + while (cursor != null) { + HttpResponse response; + String citationsUriWithCursor = citationsUri + "&per-page=" + perPage + "&cursor=" + cursor; + if (email == null || email.isBlank()) { + response = Utils.getAsHttpResponse(citationsUriWithCursor); + } else { + response = Utils.getAsHttpResponse(citationsUriWithCursor, "User-Agent", "mailto:" + email); + } + JsonObject tree = JsonParser.parseString(response.body()).getAsJsonObject(); + + cursor = Utils.stringOrNull(tree + .getAsJsonObject("meta") + .get("next_cursor") + ); + + JsonArray citationsArray = tree + .getAsJsonArray("results"); + + for (JsonElement citation : citationsArray) { + ExternalMentionRecord citationAsMention; + try { + citationAsMention = parseAsMention(citation); + } catch (RuntimeException e) { + Utils.saveExceptionInDatabase("Citation scraper", "mention", id, e); + continue; + } + citations.add(citationAsMention); + } + } + + return citations; + } + + static ExternalMentionRecord parseAsMention(JsonElement element) { + JsonObject citationObject = element.getAsJsonObject(); + + String doiUrl = Utils.stringOrNull(citationObject.get("doi")); + String doiString = doiUrl; + if (doiString != null) { + doiString = doiString.replace("https://doi.org/", ""); + } + Doi doi = doiString == null ? null : Doi.fromString(doiString); + + URI url; + if (doiUrl != null) { + url = URI.create(doiUrl); + } else { + JsonArray locations = citationObject.getAsJsonArray("locations"); + url = extractUrlFromLocation(locations); + } + + String title = Utils.stringOrNull(citationObject.get("title")); + if (title == null) { + String openAlexId = citationObject.getAsJsonPrimitive("id").getAsString(); + String message = "The title of the mention with DOI %s and OpenAlex ID %s is null".formatted(doiString, openAlexId); + throw new RuntimeException(message); + } + + JsonArray authorsArray = citationObject.getAsJsonArray("authorships"); + String authors = StreamSupport.stream(authorsArray.spliterator(), false) + .map(JsonElement::getAsJsonObject) + .map(jo -> jo.get("raw_author_name")) + .filter(Predicate.not(JsonElement::isJsonNull)) + .map(JsonElement::getAsString) + .collect(Collectors.joining(", ")); + if (authors.isBlank()) { + authors = null; + } + + Integer publicationYear = Utils.integerOrNull(citationObject.get("publication_year")); + + String crossrefType = Utils.stringOrNull(citationObject.get("type_crossref")); + MentionType mentionType = CrossrefMention.crossrefTypeMap.getOrDefault(crossrefType, MentionType.other); + + String openalexIdString = citationObject + .getAsJsonObject("ids") + .getAsJsonPrimitive("openalex") + .getAsString(); + OpenalexId openalexId = OpenalexId.fromString(openalexIdString); + + return new ExternalMentionRecord( + doi, + null, + openalexId, + url, + title, + authors, + null, + publicationYear, + null, + null, + mentionType, + "OpenAlex", + null + ); + } + + static URI extractUrlFromLocation(JsonArray locations) { + if (locations == null) { + return null; + } + + for (JsonElement location : locations) { + JsonObject locationObject = location.getAsJsonObject(); + String landingPageUrl = Utils.stringOrNull(locationObject.get("landing_page_url")); + if (landingPageUrl != null) { + landingPageUrl = landingPageUrl.replace("\\", "%5C"); + return URI.create(landingPageUrl); + } + + String pdfUrl = Utils.stringOrNull(locationObject.get("pdf_url")); + if (pdfUrl != null) { + return URI.create(pdfUrl); + } + } + + return null; + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenalexId.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenalexId.java new file mode 100644 index 0000000..03ac679 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenalexId.java @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import nl.esciencecenter.rsd.scraper.Utils; + +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +// https://docs.openalex.org/how-to-use-the-api/get-single-entities#the-openalex-id +public class OpenalexId { + + private static final Pattern OPENALEX_PATTERN = Pattern.compile("^https://openalex\\.org/([WwAaSsIiCcPpFf]\\d{3,13})$"); + private static final String OPENALEX_ID_BASE = "https://openalex.org/"; + + private final String openalexKey; + + private OpenalexId(String openalexString) { + this.openalexKey = openalexString.toUpperCase(Locale.ROOT); + } + + public static boolean isValid(String idToTest) { + return idToTest != null && OPENALEX_PATTERN.asPredicate().test(idToTest); + } + + public static OpenalexId fromString(String id) { + if (id == null) { + throw new IllegalArgumentException("The ID cannot be null"); + } + Matcher matcher = OPENALEX_PATTERN.matcher(id); + + if (!matcher.find()) { + throw new IllegalArgumentException("This is an invalid OpenAlex ID"); + } + + String key = matcher.group(1); + return new OpenalexId(key); + } + + public String toUrlEncodedString() { + return Utils.urlEncode(toString()); + } + + public String getOpenalexKey() { + return openalexKey; + } + + @Override + public String toString() { + return OPENALEX_ID_BASE + openalexKey; + } + + @Override + public int hashCode() { + return openalexKey.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (this == other) { + return true; + } + if (other instanceof OpenalexId otherOpenalexId) { + return openalexKey.equals(otherOpenalexId.openalexKey); + } + + return false; + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestCitationRepository.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestCitationRepository.java new file mode 100644 index 0000000..cad4b1b --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestCitationRepository.java @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.Utils; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Objects; +import java.util.UUID; + +/** + * This class provides access to the citation related tables via the Postgrest API. + */ +public class PostgrestCitationRepository { + + // The base URL of the backend. + private final String backendUrl; + + public PostgrestCitationRepository(String backendUrl) { + this.backendUrl = Objects.requireNonNull(backendUrl); + } + + /** + * Retrieve the least recently scraped reference papers from the database. + * + * @param limit the maximum number of references to return + * @return A collection of citation data representing these reference papers. + */ + public Collection leastRecentlyScrapedCitations(int limit) { + String oneHourAgoFilter = Utils.atLeastOneHourAgoFilter("citations_scraped_at"); + String uri = backendUrl + "/rpc/reference_papers_to_scrape?order=citations_scraped_at.asc.nullsfirst&limit=" + limit + "&" + oneHourAgoFilter; + String data = Utils.getAsAdmin(uri); + return parseJson(data); + } + + public void saveCitations(String backendUrl, UUID idCitedMention, Collection citingMentions, Instant scrapedAt) { + String jsonPatch = "{\"citations_scraped_at\": \"%s\"}".formatted(scrapedAt.toString()); + Utils.patchAsAdmin(backendUrl + "/mention?id=eq." + idCitedMention.toString(), jsonPatch); + + JsonArray jsonArray = new JsonArray(); + + // We sometimes encounter duplicate citations which may lead to the operation to fail. + HashSet seen = new HashSet<>(); + + for (UUID citingMention : citingMentions) { + + if (citingMention != null) { + String citationID = citingMention.toString(); + + if (!seen.contains(citationID)) { + seen.add(citationID); + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("mention", idCitedMention.toString()); + jsonObject.addProperty("citation", citationID); + jsonArray.add(jsonObject); + } + } + } + + String uri = backendUrl + "/citation_for_mention"; + + Utils.postAsAdmin(uri, jsonArray.toString(), "Prefer", "resolution=merge-duplicates"); + } + + private Collection parseJson(String data) { + + JsonArray array = JsonParser.parseString(data).getAsJsonArray(); + Collection result = new ArrayList<>(); + + for (JsonElement jsonElement : array) { + JsonObject jsonObject = jsonElement.getAsJsonObject(); + UUID id = UUID.fromString(jsonObject.getAsJsonPrimitive("id").getAsString()); + String doiString = Utils.stringOrNull(jsonObject.get("doi")); + Doi doi = doiString == null ? null : Doi.fromString(doiString); + String openalexIdString = Utils.stringOrNull(jsonObject.get("openalex_id")); + OpenalexId openalexId = openalexIdString == null ? null : OpenalexId.fromString(openalexIdString); + + Collection knownDois = new HashSet<>(); + JsonArray doisArray = jsonObject.getAsJsonArray("known_citing_dois"); + + for (JsonElement element : doisArray) { + knownDois.add(Doi.fromString(element.getAsString())); + } + + result.add(new CitationData(id, doi, openalexId, knownDois)); + } + + return result; + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestMentionRepository.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestMentionRepository.java new file mode 100644 index 0000000..37e62b6 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestMentionRepository.java @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import com.google.gson.FieldNamingPolicy; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializer; +import nl.esciencecenter.rsd.scraper.Utils; + +import java.net.URI; +import java.time.Instant; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Objects; +import java.util.UUID; + +public class PostgrestMentionRepository { + + private static final Gson GSON = new GsonBuilder() + .serializeNulls() + .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES) + .registerTypeAdapter(Instant.class, (JsonSerializer) (src, typeOfSrc, context) -> new JsonPrimitive(src.toString())) + .registerTypeAdapter(ZonedDateTime.class, (JsonSerializer) (src, typeOfSrc, context) -> new JsonPrimitive(src.toString())) + .registerTypeAdapter(Doi.class, (JsonSerializer) (src, typeOfSrc, context) -> new JsonPrimitive(src.toString())) + .create(); + + private final String backendUrl; + + + public PostgrestMentionRepository(String backendUrl) { + this.backendUrl = Objects.requireNonNull(backendUrl); + } + + static Collection parseMultipleRsdIds(String json) { + Collection result = new ArrayList<>(); + + JsonArray rootArray = JsonParser.parseString(json).getAsJsonArray(); + for (JsonElement jsonElement : rootArray) { + + JsonObject arrayEntry = jsonElement.getAsJsonObject(); + UUID id = UUID.fromString(arrayEntry.getAsJsonPrimitive("id").getAsString()); + String doiString = Utils.stringOrNull(arrayEntry.get("doi")); + Doi doi = doiString == null ? null : Doi.fromString(doiString); + String openalexIdString = Utils.stringOrNull(arrayEntry.get("openalex_id")); + OpenalexId openalexId = openalexIdString == null ? null : OpenalexId.fromString(openalexIdString); + + result.add(new RsdMentionIds(id, doi, openalexId)); + } + + return result; + } + + public static JsonArray toRsdJsonArray(Collection mentions) { + return GSON.toJsonTree(mentions).getAsJsonArray(); + } + + static RsdMentionIds parseSingleRsdIds(String json) { + JsonObject root = JsonParser.parseString(json).getAsJsonArray().get(0).getAsJsonObject(); + + UUID id = UUID.fromString(root.getAsJsonPrimitive("id").getAsString()); + String doiString = Utils.stringOrNull(root.get("doi")); + Doi doi = doiString == null ? null : Doi.fromString(doiString); + String openalexIdString = Utils.stringOrNull(root.get("openalex_id")); + OpenalexId openalexId = openalexIdString == null ? null : OpenalexId.fromString(openalexIdString); + + return new RsdMentionIds(id, doi, openalexId); + } + + public Collection leastRecentlyScrapedMentions(int limit) { + String data = Utils.getAsAdmin(backendUrl + "/mention?or=(doi.not.is.null,openalex_id.not.is.null)&order=scraped_at.asc.nullsfirst&select=id,doi,openalex_id&limit=" + limit); + return parseMultipleRsdIds(data); + } + + public void saveScrapedAt(RsdMentionIds ids, Instant scrapedAt) { + JsonObject root = new JsonObject(); + root.addProperty("scraped_at", scrapedAt.toString()); + Utils.patchAsAdmin(backendUrl + "/mention?select=id,doi,openalex_id&id=eq." + ids.id(), root.toString(), "Prefer", "return=representation"); + } + + public RsdMentionIds updateMention(RsdMentionRecord mention, boolean updateOpenAlexId) { + JsonObject root = createJsonFromMentionData(mention.content(), updateOpenAlexId); + root.addProperty("scraped_at", mention.scrapedAt().toString()); + String response = Utils.patchAsAdmin(backendUrl + "/mention?select=id,doi,openalex_id&id=eq." + mention.id(), root.toString(), "Prefer", "return=representation"); + return parseSingleRsdIds(response); + } + + public RsdMentionIds createMentionIfNotExistsOnDoiAndGetIds(ExternalMentionRecord mention, Instant scrapedAt) { + Doi doi = mention.doi(); + Objects.requireNonNull(doi); + Collection mentionsWithDoi = parseMultipleRsdIds(Utils.getAsAdmin(backendUrl + "/mention?select=id,doi,openalex_id&doi=eq." + doi.toUrlEncodedString())); + if (mentionsWithDoi.size() == 1) { + return mentionsWithDoi.iterator().next(); + } + + return createNewMention(mention, scrapedAt, false); + } + + public RsdMentionIds createOrUpdateMentionWithOpenalexId(ExternalMentionRecord mention, Instant scrapedAt) { + OpenalexId openalexId = Objects.requireNonNull(mention.openalexId()); + Doi doi = mention.doi(); + + String query = "/mention?select=id,doi,openalex_id"; + if (mention.doi() != null) { + query += "&or=(openalex_id.eq.%s,doi.eq.%s)".formatted(openalexId.toUrlEncodedString(), doi.toUrlEncodedString()); + } else { + query += "&openalex_id=eq.%s".formatted(openalexId.toUrlEncodedString()); + } + String existingMentionsResponse = Utils.getAsAdmin(backendUrl + query); + Collection existingIds = parseMultipleRsdIds(existingMentionsResponse); + + if (existingIds.size() > 1) { + throw new RuntimeException("Multiple entries with DOI %s or OpenAlex id %s exist, they should be merged".formatted(openalexId, mention.doi())); + } + if (existingIds.size() == 1) { + UUID id = existingIds.iterator().next().id(); + return updateMention(new RsdMentionRecord(id, mention, scrapedAt), true); + } + + return createNewMention(mention, scrapedAt, true); + } + + private RsdMentionIds createNewMention(ExternalMentionRecord mention, Instant scrapedAt, boolean setOpenAlexId) { + JsonObject root = createJsonFromMentionData(mention, setOpenAlexId); + root.addProperty("scraped_at", scrapedAt.toString()); + String response = Utils.postAsAdmin(backendUrl + "/mention?select=id,doi,openalex_id", root.toString(), "Prefer", "return=representation"); + return parseSingleRsdIds(response); + } + + static JsonObject createJsonFromMentionData(ExternalMentionRecord mention, boolean setOpenAlexId) { + JsonObject root = new JsonObject(); + Doi doi = mention.doi(); + root.addProperty("doi", doi == null ? null : mention.doi().toString()); + ZonedDateTime doiRegistrationDate = mention.doiRegistrationDate(); + root.addProperty("doi_registration_date", doiRegistrationDate == null ? null : doiRegistrationDate.toString()); + if (setOpenAlexId) { + root.addProperty("openalex_id", mention.openalexId().toString()); + } + URI url = mention.url(); + root.addProperty("url", url == null ? null : mention.url().toString()); + root.addProperty("title", mention.title()); + root.addProperty("authors", mention.authors()); + root.addProperty("publisher", mention.publisher()); + root.addProperty("publication_year", mention.publicationYear()); + root.addProperty("journal", mention.journal()); + root.addProperty("page", mention.page()); + root.addProperty("mention_type", mention.mentionType().name()); + root.addProperty("source", mention.source()); + root.addProperty("version", mention.version()); + root.addProperty("source", mention.source()); + + return root; + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestReleaseRepository.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestReleaseRepository.java new file mode 100644 index 0000000..51e13c3 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestReleaseRepository.java @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import com.google.gson.FieldNamingPolicy; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonObject; +import com.google.gson.reflect.TypeToken; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.Utils; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; + +public class PostgrestReleaseRepository { + + private final String backendUrl; + + public PostgrestReleaseRepository(String backendUrl) { + this.backendUrl = Objects.requireNonNull(backendUrl); + } + + public Collection leastRecentlyScrapedReleases(int limit) { + String data = Utils.getAsAdmin(backendUrl + "/rpc/software_join_release?concept_doi=not.is.null&order=releases_scraped_at.asc.nullsfirst&limit=" + limit); + return parseJson(data); + } + + public void saveReleaseContent(Collection releaseData, Map> conceptDoiToDois, Map versionDoiToMentionId) { + // First update the releases_scraped_at column. + JsonArray releasesBody = new JsonArray(); + Instant now = Instant.now(); + for (ReleaseData release : releaseData) { + JsonObject data = new JsonObject(); + data.addProperty("software", release.softwareId.toString()); + data.addProperty("releases_scraped_at", now.toString()); + releasesBody.add(data); + } + Utils.postAsAdmin(Config.backendBaseUrl() + "/release", releasesBody.toString(), "Prefer", "resolution=merge-duplicates"); + + + // Then update the release_version table. + // For each scraped or existing version as a mention, we need to know its id of the mention table and the ids (plural, because multiple software entries can have the same concept DOI) of the software to which it belongs. + Map> conceptDoiToSoftwareIds = new HashMap<>(); + for (ReleaseData release : releaseData) { + Collection softwareIds = conceptDoiToSoftwareIds.computeIfAbsent(release.conceptDoi, k -> new ArrayList<>()); + softwareIds.add(release.softwareId); + } + + Map versionDoiToConceptDoi = new HashMap<>(); + for (Map.Entry> conceptDoiToDoisEntry : conceptDoiToDois.entrySet()) { + Doi conceptDoi = conceptDoiToDoisEntry.getKey(); + Collection versionDois = conceptDoiToDoisEntry.getValue(); + for (ExternalMentionRecord version : versionDois) { + versionDoiToConceptDoi.put(version.doi(), conceptDoi); + } + } + + JsonArray coupling = new JsonArray(); + for (Map.Entry entry : versionDoiToConceptDoi.entrySet()) { + Doi versionDoi = entry.getKey(); + Doi conceptDoi = entry.getValue(); + Collection softwareIds = conceptDoiToSoftwareIds.get(conceptDoi); + for (UUID softwareId : softwareIds) { + JsonObject couple = new JsonObject(); + UUID mentionId = versionDoiToMentionId.get(versionDoi); + if (mentionId == null) { + continue; + } + couple.addProperty("release_id", softwareId.toString()); + couple.addProperty("mention_id", mentionId.toString()); + coupling.add(couple); + } + } + + Utils.postAsAdmin(Config.backendBaseUrl() + "/release_version", coupling.toString(), "Prefer", "resolution=merge-duplicates"); + } + + Collection parseJson(String data) { + JsonDeserializer doiDeserializer = (json, type, context) -> Doi.fromString(json.getAsJsonPrimitive().getAsString()); + Gson gson = new GsonBuilder() + .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES) + .registerTypeAdapter(Doi.class, doiDeserializer) + .create(); + TypeToken> typeToken = new TypeToken<>() { + }; + return gson.fromJson(data, typeToken.getType()); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ReleaseData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ReleaseData.java new file mode 100644 index 0000000..55807be --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ReleaseData.java @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import java.util.Collection; +import java.util.UUID; + +public class ReleaseData { + + public UUID softwareId; + public String slug; + public Doi conceptDoi; + public Collection versionedDois; + + @Override + public String toString() { + return "ReleaseData{" + + "softwareId=" + softwareId + + ", slug='" + slug + '\'' + + ", conceptDoi='" + conceptDoi + '\'' + + ", versionedDois=" + versionedDois + + '}'; + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionIds.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionIds.java new file mode 100644 index 0000000..81d529e --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionIds.java @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import java.util.UUID; + +public record RsdMentionIds( + UUID id, + Doi doi, + OpenalexId openalexId +) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionRecord.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionRecord.java new file mode 100644 index 0000000..237e961 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionRecord.java @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import java.time.Instant; +import java.util.UUID; + +public record RsdMentionRecord( + UUID id, + ExternalMentionRecord content, + Instant scrapedAt +) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitData.java new file mode 100644 index 0000000..2c93ed9 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitData.java @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +public record BasicGitData( + Boolean archived, + String license, + Long starCount, + Integer forkCount, + Integer openIssueCount +) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitDatabaseData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitDatabaseData.java new file mode 100644 index 0000000..3374cc1 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitDatabaseData.java @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2023 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import java.time.ZonedDateTime; + +public record BasicGitDatabaseData( + BasicRepositoryData basicData, + BasicGitData statsData, + ZonedDateTime dataScrapedAt +) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryData.java new file mode 100644 index 0000000..161fe4f --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryData.java @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2023 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import java.util.UUID; + +public record BasicRepositoryData(UUID software, String url) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryDataWithHistory.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryDataWithHistory.java new file mode 100644 index 0000000..39c4863 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryDataWithHistory.java @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: 2025 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import java.time.ZonedDateTime; +import java.util.UUID; + +public record BasicRepositoryDataWithHistory(UUID software, String url, ZonedDateTime commitHistoryScrapedAt, CommitsPerWeek commitsPerWeek) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CodePlatformProvider.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CodePlatformProvider.java new file mode 100644 index 0000000..6ee6449 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CodePlatformProvider.java @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +public enum CodePlatformProvider { + GITHUB, GITLAB, BITBUCKET, FOURTU, OTHER +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitData.java new file mode 100644 index 0000000..dbe03ab --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitData.java @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import java.time.ZonedDateTime; + +public record CommitData( + BasicRepositoryData basicData, + CommitsPerWeek commitHistory, + ZonedDateTime commitHistoryScrapedAt +) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeek.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeek.java new file mode 100644 index 0000000..c8eae75 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeek.java @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: 2023 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 Netherlands eScience Center +// SPDX-FileCopyrightText: 2025 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializer; + +import java.time.DayOfWeek; +import java.time.Instant; +import java.time.Period; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAdjusters; +import java.util.SortedMap; +import java.util.TreeMap; + +/** + * Aggregates commits per week. We choose a week to begin on Sunday at midnight UTC, since GitHub aggregates commits at this instance too. + * In the underlying map, we choose Sunday midnight UTC as a representative for a week as well. + */ +public class CommitsPerWeek { + + private final SortedMap data = new TreeMap<>(); + static final Gson gson = new GsonBuilder() + .enableComplexMapKeySerialization() + .registerTypeAdapter(Instant.class, (JsonSerializer) (src, typeOfSrc, context) -> new JsonPrimitive(src.getEpochSecond())) + .create(); + + SortedMap getData() { + return new TreeMap<>(data); + } + + public void addCommits(ZonedDateTime zonedDateTime, long count) { + ZonedDateTime utcTime = zonedDateTime.withZoneSameInstant(ZoneOffset.UTC); + Instant sundayMidnight = utcTime.truncatedTo(ChronoUnit.DAYS).with(TemporalAdjusters.previousOrSame(DayOfWeek.SUNDAY)).toInstant(); + data.merge(sundayMidnight, count, Long::sum); + } + + public void addCommits(Instant instant, long count) { + ZonedDateTime utcTime = ZonedDateTime.ofInstant(instant, ZoneOffset.UTC); + addCommits(utcTime, count); + } + + public ZonedDateTime popLatestTimestamp() { + if (this.data.size() > 0) { + Instant lastTimeStamp = this.data.lastKey(); + this.data.remove(lastTimeStamp); + return ZonedDateTime.ofInstant(lastTimeStamp, ZoneId.systemDefault()); + } + return null; + } + + /** + * Checks every week from the first week present to the last week. If a week is not present, puts zero for that week. + * We need these zero values so that the commit graphs are displayed properly. + */ + public void addMissingZeros() { + if (data.isEmpty()) return; + + Instant firstWeek = data.firstKey(); + Instant lastWeek = data.lastKey(); + + for (Instant currentWeek = firstWeek; currentWeek.isBefore(lastWeek); currentWeek = currentWeek.plus(Period.ofWeeks(1))) { + data.putIfAbsent(currentWeek, 0L); + } + } + + public String toJson() { + return gson.toJson(data); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/ContributorDatabaseData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/ContributorDatabaseData.java new file mode 100644 index 0000000..e0eb6bb --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/ContributorDatabaseData.java @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2023 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import java.time.ZonedDateTime; + +public record ContributorDatabaseData( + BasicRepositoryData basicData, + Integer contributorCount, + ZonedDateTime dataScrapedAt +) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/FourTuGitScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/FourTuGitScraper.java new file mode 100644 index 0000000..65ebe3b --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/FourTuGitScraper.java @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; + +import java.io.IOException; +import java.net.http.HttpResponse; +import java.util.Objects; + +public class FourTuGitScraper implements GitScraper { + private final String repoUrl; + + public FourTuGitScraper(String repoUrl) { + this.repoUrl = Objects.requireNonNull(repoUrl); + } + + @Override + public BasicGitData basicData() throws IOException, InterruptedException, RsdResponseException { + throw new UnsupportedOperationException(); + } + + // Example URL: https://data.4tu.nl/v3/datasets/454afe4e-08aa-4333-b6e8-7db7cebd0f83.git/languages + @Override + public String languages() throws IOException, InterruptedException, RsdResponseException { + String url = repoUrl + "/languages"; + HttpResponse response = Utils.getAsHttpResponse(url); + return switch (response.statusCode()) { + case 404 -> + throw new RsdResponseException(404, response.uri(), response.body(), "Not found, is the repository URL correct?"); + case 200 -> response.body(); + default -> + throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response when downloading 4TU programming languages"); + }; + } + + // Example URL: https://data.4tu.nl/v3/datasets/454afe4e-08aa-4333-b6e8-7db7cebd0f83.git/contributors + @Override + public CommitsPerWeek contributions() throws IOException, InterruptedException, RsdResponseException { + String url = repoUrl + "/contributors"; + HttpResponse response = Utils.getAsHttpResponse(url); + return switch (response.statusCode()) { + case 404 -> + throw new RsdResponseException(404, response.uri(), response.body(), "Not found, is the repository URL correct?"); + case 200 -> GithubScraper.parseCommits(response.body()); + default -> + throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response when downloading 4TU commit history"); + }; + } + + // Example URL: https://data.4tu.nl/v3/datasets/454afe4e-08aa-4333-b6e8-7db7cebd0f83.git/contributors + @Override + public Integer contributorCount() throws IOException, InterruptedException, RsdResponseException { + String url = repoUrl + "/contributors"; + HttpResponse response = Utils.getAsHttpResponse(url); + return switch (response.statusCode()) { + case 404 -> + throw new RsdResponseException(404, response.uri(), response.body(), "Not found, is the repository URL correct?"); + case 200 -> JsonParser.parseString(response.body()).getAsJsonArray().size(); + default -> + throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response when downloading 4TU commit history"); + }; + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitScraper.java new file mode 100644 index 0000000..31dc31c --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitScraper.java @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2022 - 2023 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2023 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import java.io.IOException; + +import nl.esciencecenter.rsd.scraper.RsdResponseException; + +public interface GitScraper { + + BasicGitData basicData() throws IOException, InterruptedException, RsdResponseException; + + String languages() throws IOException, InterruptedException, RsdResponseException; + + CommitsPerWeek contributions() throws IOException, InterruptedException, RsdResponseException; + + Integer contributorCount() throws IOException, InterruptedException, RsdResponseException; +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GithubScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GithubScraper.java new file mode 100644 index 0000000..428c0f3 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GithubScraper.java @@ -0,0 +1,240 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdRateLimitException; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; + +import java.io.IOException; +import java.net.http.HttpResponse; +import java.time.Instant; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class GithubScraper implements GitScraper { + + private static final String BASE_API_URL = "https://api.github.com"; + public final String organisation; + public final String repo; + private static final Pattern LINK_PATTERN = Pattern.compile("<([^>]+page=(\\d+)[^>]*)>; rel=\"([^\"]+)\""); + private static final Pattern GITHUB_URL_PATTERN = Pattern.compile("^https?://github\\.com/([^\\s/]+)/([^\\s/]+)/?$"); + + private GithubScraper(String organisation, String repo) { + this.organisation = Objects.requireNonNull(organisation); + if (repo.endsWith(".git")) { + repo = repo.substring(0, repo.length() - 4); + } + this.repo = Objects.requireNonNull(repo); + } + + public static Optional create(String url) { + Matcher matcher = GITHUB_URL_PATTERN.matcher(url); + if (!matcher.find()) { + return Optional.empty(); + } + + String organisation = matcher.group(1); + String repo = matcher.group(2); + GithubScraper githubScraper = new GithubScraper(organisation, repo); + return Optional.of(githubScraper); + } + + /** + * Returns the basic data of the repository. + * Example URL: https://api.github.com/repos/research-software-directory/RSD-as-a-service + */ + @Override + public BasicGitData basicData() throws IOException, InterruptedException, RsdResponseException { + Optional apiCredentials = Config.apiCredentialsGithub(); + HttpResponse response; + if (apiCredentials.isPresent()) { + response = Utils.getAsHttpResponse(BASE_API_URL + "/repos/" + organisation + "/" + repo, "Authorization", "Basic " + Utils.base64Encode(apiCredentials.get())); + } else { + response = Utils.getAsHttpResponse(BASE_API_URL + "/repos/" + organisation + "/" + repo); + } + return switch (response.statusCode()) { + case 200 -> parseBasicData(response.body()); + case 404 -> + throw new RsdResponseException(404, response.uri(), response.body(), "Not found, is the repository URL correct?"); + case 403 -> + throw new RsdRateLimitException(403, response.uri(), response.body(), "Rate limit for GitHub probably reached"); + default -> + throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response"); + }; + } + + /** + * Returns JSON as a String with the amount of lines written in each language. + * Example URL: https://api.github.com/repos/research-software-directory/RSD-as-a-service/languages + */ + @Override + public String languages() throws IOException, InterruptedException, RsdResponseException { + HttpResponse response = getAsHttpResponse(BASE_API_URL + "/repos/" + organisation + "/" + repo + "/languages"); + return switch (response.statusCode()) { + case 404 -> + throw new RsdResponseException(404, response.uri(), response.body(), "Not found, is the repository URL correct?"); + case 403 -> + throw new RsdRateLimitException(403, response.uri(), response.body(), "Rate limit for GitHub probably reached"); + case 200 -> response.body(); + default -> + throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response"); + }; + } + + /** + * Returns all contributors commit activity. + * https://docs.github.com/en/rest/reference/metrics#get-all-contributor-commit-activity= + * Requesting commit activity requires a GitHub authentication token. + *

+ * The returned string represents a JsonArray with one entry per contributor. THe information + * per entry are: + *

+ * { + * "author": { }, + * "total": number of total commits, + * "weeks": [ + * { + * "w": unix timestamp (Start of the week 00:00 on Sundays), + * "a": number of additions, + * "d": number of deletions, + * "c": number of commits + * }, ... + * ] + * } + *

+ * Example URL: https://api.github.com/repos/research-software-directory/RSD-as-a-service/stats/contributors + */ + @Override + public CommitsPerWeek contributions() throws IOException, InterruptedException, RsdResponseException { + HttpResponse httpResponse = null; + for (int i = 0; i < 2; i++) { + httpResponse = getAsHttpResponse(BASE_API_URL + "/repos/" + organisation + "/" + repo + "/stats/contributors"); + + if (httpResponse.statusCode() != 202) break; + Thread.sleep(3000L); + } + + int status = httpResponse.statusCode(); + if (status == 404) { + throw new RsdResponseException(status, httpResponse.uri(), httpResponse.body(), "Not found, is the repository URL correct?"); + } else if (status == 204) { + // empty commit history + return new CommitsPerWeek(); + } else if (status == 403) { + throw new RsdRateLimitException(403, httpResponse.uri(), httpResponse.body(), "Rate limit for GitHub probably reached"); + } else if (status == 202) { + // response not ready yet + return null; + } else if (status != 200) { + throw new RsdResponseException(status, httpResponse.uri(), httpResponse.body(), "Unexpected response"); + } else { + String contributionsJson = httpResponse.body(); + return parseCommits(contributionsJson); + } + } + + // Example URL: https://api.github.com/repos/research-software-directory/RSD-as-a-service/contributors?per_page=1 + @Override + public Integer contributorCount() throws IOException, InterruptedException, RsdResponseException { + // we request one contributor per page and just extract the number of pages from the headers + // see https://docs.github.com/en/rest/guides/using-pagination-in-the-rest-api?apiVersion=2022-11-28 + HttpResponse httpResponse = getAsHttpResponse(BASE_API_URL + "/repos/" + organisation + "/" + repo + "/contributors?per_page=1"); + + int status = httpResponse.statusCode(); + if (status == 404) { + throw new RsdResponseException(404, httpResponse.uri(), httpResponse.body(), "Not found, is the repository URL correct?"); + } else if (status == 403) { + throw new RsdRateLimitException(403, httpResponse.uri(), httpResponse.body(), "Rate limit for GitHub probably reached"); + } else if (status != 200) { + throw new RsdResponseException(status, httpResponse.uri(), httpResponse.body(), "Unexpected response"); + } else { + List linkHeaders = httpResponse.headers().allValues("link"); + String[] lastPageData = lastPageFromLinkHeader(linkHeaders); + if (lastPageData != null) { + int lastPageNumber = Integer.parseInt(lastPageData[1]); + return lastPageNumber; + } else { + // this was the first page, return the size of the array (either 0 or 1) + return JsonParser.parseString(httpResponse.body()).getAsJsonArray().size(); + } + } + } + + static CommitsPerWeek parseCommits(String json) { + CommitsPerWeek commits = new CommitsPerWeek(); + JsonArray commitsPerContributor = JsonParser.parseString(json).getAsJsonArray(); + + for (JsonElement jsonElement : commitsPerContributor) { + JsonArray weeks = jsonElement.getAsJsonObject().getAsJsonArray("weeks"); + for (JsonElement week : weeks) { + long weekTimestamp = week.getAsJsonObject().getAsJsonPrimitive("w").getAsLong(); + long commitsInWeek = week.getAsJsonObject().getAsJsonPrimitive("c").getAsLong(); + + Instant weekInstant = Instant.ofEpochSecond(weekTimestamp); + + commits.addCommits(weekInstant, commitsInWeek); + } + } + + return commits; + } + + static BasicGitData parseBasicData(String json) { + JsonObject jsonObject = JsonParser.parseString(json).getAsJsonObject(); + + Boolean archived = jsonObject.getAsJsonPrimitive("archived").getAsBoolean(); + JsonElement jsonLicense = jsonObject.get("license"); + String license = jsonLicense.isJsonNull() ? null : jsonLicense.getAsJsonObject() + .getAsJsonPrimitive("spdx_id") + .getAsString(); + Long starCount = jsonObject.getAsJsonPrimitive("stargazers_count").getAsLong(); + Integer forkCount = jsonObject.getAsJsonPrimitive("forks_count").getAsInt(); + Integer openIssueCount = jsonObject.getAsJsonPrimitive("open_issues_count").getAsInt(); + + return new BasicGitData( + archived, + license, + starCount, + forkCount, + openIssueCount + ); + } + + // return an object with the URL of the last page and the number of the last page respectively + static String[] lastPageFromLinkHeader(List links) { + if (links.isEmpty()) return null; + + for (String link : links) { + Matcher matcher = LINK_PATTERN.matcher(link); + while (matcher.find()) { + String relation = matcher.group(3); + if (relation.equals("last")) return new String[]{matcher.group(1), matcher.group(2)}; + } + } + + throw new RuntimeException("No last page found"); + } + + static HttpResponse getAsHttpResponse(String url) throws IOException, InterruptedException { + Optional apiCredentials = Config.apiCredentialsGithub(); + if (apiCredentials.isPresent()) { + return Utils.getAsHttpResponse(url, "Authorization", "Basic " + Utils.base64Encode(apiCredentials.get())); + } else { + return Utils.getAsHttpResponse(url); + } + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitlabScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitlabScraper.java new file mode 100644 index 0000000..479a090 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitlabScraper.java @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2025 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2022 - 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2025 Paula Stock (GFZ) +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.time.ZonedDateTime; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import nl.esciencecenter.rsd.scraper.RsdRateLimitException; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; + +public class GitlabScraper implements GitScraper { + + private final String projectPath; + private final String apiUri; + private final String graphqlUri; + private final CommitsPerWeek commitsPerWeek; + private final ZonedDateTime lastCommitHistoryTimestamp; + + /** + * A GitLab scraper for API version 4. + * + * @param gitLabApiUrl The API Url without a version, e.g. https:///api + * @param projectPath The full path to the project + */ + public GitlabScraper(String gitLabApiUrl, String projectPath) { + this.projectPath = projectPath.endsWith(".git") ? projectPath.substring(0, projectPath.length() - 4) : projectPath; + this.apiUri = gitLabApiUrl + "/v4"; + this.graphqlUri = gitLabApiUrl + "/graphql"; + this.lastCommitHistoryTimestamp = null; + this.commitsPerWeek = new CommitsPerWeek(); + } + + public GitlabScraper(String gitLabApiUrl, String projectPath, ZonedDateTime lastCommitHistoryTimestamp, CommitsPerWeek existingCommitsPerWeek) { + this.projectPath = projectPath.endsWith(".git") ? projectPath.substring(0, projectPath.length() - 4) : projectPath; + this.apiUri = gitLabApiUrl + "/v4"; + this.graphqlUri = gitLabApiUrl + "/graphql"; + this.lastCommitHistoryTimestamp = lastCommitHistoryTimestamp; + this.commitsPerWeek = existingCommitsPerWeek == null ? new CommitsPerWeek() : existingCommitsPerWeek; + } + + /** + * Returns the basic data of a project. If GitLab detects the license, an SPDX identifier will be + * returned. If the license could not be detected, returns "Other". API endpoint: + * https://docs.gitlab.com/ee/api/projects.html#get-single-project NOTE: A GraphQL request here + * might be more efficient since less data would be sent. + * Example URL: https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab-shell?license=True + * + * @return The basic data + */ + @Override + public BasicGitData basicData() throws IOException, InterruptedException, RsdResponseException { + String response = Utils.get(apiUri + "/projects/" + Utils.urlEncode(projectPath) + "?license=True"); + + return parseBasicData(response, checkIfArchived()); + } + + /** + * Returns the languages used in a project with percentage values. Uses the API Endpoint + * https://docs.gitlab.com/ee/api/projects.html#languages GET /projects/:id/languages + *

+ * Example URL: https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab-shell/languages + * + * @return A JSON as a String + */ + @Override + public String languages() throws IOException, InterruptedException, RsdResponseException { + return Utils.get(apiUri + "/projects/" + Utils.urlEncode(projectPath) + "/languages"); + } + + /** + * Retrieve all contributions from GitLab API. Uses the API Endpoint: + * https://docs.gitlab.com/ee/api/commits.html#list-repository-commits + * This endpoint does not require authentication for public projects. + *

+ * Maybe use GraphQL to reduce return data? + * https://docs.gitlab.com/ee/api/graphql/reference/#commit= + * + * @return String representing a JSON array with the defined metadata (id, committed_date). + *

+ * Example URL: https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab-shell/repository/commits?per_page=100&order=default&page=1 + */ + @Override + public CommitsPerWeek contributions() throws IOException, InterruptedException, RsdResponseException { + String since=""; + if (lastCommitHistoryTimestamp != null) { + since = "&since=" + lastCommitHistoryTimestamp.toString(); + } + String page = "1"; + boolean done = false; + while (!done) { + HttpRequest request = HttpRequest.newBuilder().GET() + .uri(URI.create(apiUri + "/projects/" + Utils.urlEncode(projectPath) + + "/repository/commits?per_page=100&order=default&page=" + page + since)) + .timeout(Duration.ofSeconds(30)) + .build(); + HttpResponse response; + try (HttpClient client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build()) { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } + if (response.statusCode() == 429) + throw new RsdRateLimitException(429, response.uri(), response.body(), "API rate limit reached for GitLab"); + if (response.statusCode() == 404) + throw new RsdResponseException(404, response.uri(), response.body(), "Not found, is the repository URL correct?"); + + String body = response.body(); + parseCommitPage(body, commitsPerWeek); + + page = response.headers().firstValue("x-next-page").orElseThrow(); + done = page.isEmpty(); + } + return commitsPerWeek; + } + + // Example URL: https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab-shell/repository/contributors + @Override + public Integer contributorCount() throws IOException, InterruptedException, RsdResponseException { + HttpResponse httpResponse = Utils.getAsHttpResponse(apiUri + "/projects/" + Utils.urlEncode(projectPath) + "/repository/contributors"); + + if (httpResponse.statusCode() == 429) + throw new RsdRateLimitException(429, httpResponse.uri(), httpResponse.body(), "Rate limit reached for GitLab"); + if (httpResponse.statusCode() == 404) + throw new RsdResponseException(404, httpResponse.uri(), httpResponse.body(), "Not found, is the repository URL correct?"); + + // see https://docs.gitlab.com/ee/api/rest/index.html#other-pagination-headers + String totalItemsHeader = httpResponse.headers().firstValue("x-total").orElseThrow(); + return Integer.parseInt(totalItemsHeader); + } + + static void parseCommitPage(String json, CommitsPerWeek commitsToFill) { + JsonArray thisPageCommits = JsonParser.parseString(json).getAsJsonArray(); + + for (JsonElement element : thisPageCommits) { + JsonObject currentCommit = element.getAsJsonObject(); + String timeString = currentCommit.getAsJsonPrimitive("committed_date").getAsString(); + ZonedDateTime time = ZonedDateTime.parse(timeString); + + commitsToFill.addCommits(time, 1); + } + } + + private Boolean checkIfArchived() throws IOException, InterruptedException, RsdResponseException { + String graphqlQuery = String.format(""" + { + project(fullPath: "%s/%s") { + archived + } + } + """, + projectPath.substring(0, projectPath.indexOf("/")), + projectPath.substring(projectPath.indexOf("/") + 1)); + JsonObject body = new JsonObject(); + body.addProperty("query", graphqlQuery); + String response = Utils.post(graphqlUri, body.toString(), "Content-Type", "application/json"); + JsonObject jsonObject = JsonParser.parseString(response).getAsJsonObject(); + return jsonObject.getAsJsonObject("data").getAsJsonObject("project").getAsJsonPrimitive("archived").getAsBoolean(); + } + + static BasicGitData parseBasicData(String json, Boolean archived) { + JsonObject jsonObject = JsonParser.parseString(json).getAsJsonObject(); + + JsonElement jsonLicense = jsonObject.get("license"); + String license = jsonLicense.isJsonNull() ? null : jsonLicense.getAsJsonObject().get("name").getAsString(); + Long starCount = jsonObject.getAsJsonPrimitive("star_count").getAsLong(); + Integer forkCount = jsonObject.getAsJsonPrimitive("forks_count").getAsInt(); + + return new BasicGitData( + archived, + license, + starCount, + forkCount, + null + ); + } + +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/LanguagesData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/LanguagesData.java new file mode 100644 index 0000000..578ffa9 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/LanguagesData.java @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2023 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import java.time.ZonedDateTime; + +public record LanguagesData( + BasicRepositoryData basicData, + String languages, + ZonedDateTime languagesScrapedAt +) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainBasicData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainBasicData.java new file mode 100644 index 0000000..c2c3be5 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainBasicData.java @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdRateLimitException; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +public class MainBasicData { + + private static final Logger LOGGER = LoggerFactory.getLogger(MainBasicData.class); + + public static void main(String[] args) { + LOGGER.info("Start scraping basic Git data"); + + long t1 = System.currentTimeMillis(); + + scrapeGitHub(); + scrapeGitLab(); + + long time = System.currentTimeMillis() - t1; + + LOGGER.info("Done scraping basic Git data ({} ms.)", time); + } + + private static void scrapeGitHub() { + PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITHUB); + Collection dataToScrape = softwareInfoRepository.statsData(Config.maxRequestsGithub()); + CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; + ZonedDateTime scrapedAt = ZonedDateTime.now(); + int i = 0; + for (BasicRepositoryData basicData : dataToScrape) { + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + String repoUrl = basicData.url(); + + Optional githubScraperOptional = GithubScraper.create(repoUrl); + if (githubScraperOptional.isEmpty()) { + Utils.saveErrorMessageInDatabase("Not a valid GitHub URL: " + repoUrl, "repository_url", "basic_data_last_error", basicData.software().toString(), "software", scrapedAt, "basic_data_scraped_at"); + return; + } + + GithubScraper githubScraper = githubScraperOptional.get(); + BasicGitData scrapedBasicData = githubScraper.basicData(); + BasicGitDatabaseData updatedData = new BasicGitDatabaseData(basicData, scrapedBasicData, scrapedAt); + softwareInfoRepository.saveBasicData(updatedData); + } catch (RsdRateLimitException e) { + Utils.saveExceptionInDatabase("GitHub basic data scraper", "repository_url", basicData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "basic_data_last_error", basicData.software().toString(), "software", null, null); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase("GitHub basic data scraper", "repository_url", basicData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "basic_data_last_error", basicData.software().toString(), "software", scrapedAt, "basic_data_scraped_at"); + } catch (Exception e) { + Utils.saveExceptionInDatabase("GitHub basic data scraper", "repository_url", basicData.software(), e); + Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "basic_data_last_error", basicData.software().toString(), "software", scrapedAt, "basic_data_scraped_at"); + } + }); + futures[i] = future; + i++; + } + CompletableFuture.allOf(futures).join(); + } + + private static void scrapeGitLab() { + PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITLAB); + Collection dataToScrape = softwareInfoRepository.statsData(Config.maxRequestsGithub()); + CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; + ZonedDateTime scrapedAt = ZonedDateTime.now(); + int i = 0; + for (BasicRepositoryData basicData : dataToScrape) { + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + String repoUrl = basicData.url(); + String hostname = URI.create(repoUrl).getHost(); + String apiUrl = "https://" + hostname + "/api"; + String projectPath = repoUrl.replace("https://" + hostname + "/", ""); + if (projectPath.endsWith("/")) projectPath = projectPath.substring(0, projectPath.length() - 1); + + BasicGitData scrapedBasicData = new GitlabScraper(apiUrl, projectPath).basicData(); + BasicGitDatabaseData updatedData = new BasicGitDatabaseData(basicData, scrapedBasicData, scrapedAt); + softwareInfoRepository.saveBasicData(updatedData); + } catch (RsdRateLimitException e) { + Utils.saveExceptionInDatabase("GitLab basic data scraper", "repository_url", basicData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "basic_data_last_error", basicData.software().toString(), "software", null, null); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase("GitLab basic data scraper", "repository_url", basicData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "basic_data_last_error", basicData.software().toString(), "software", scrapedAt, "basic_data_scraped_at"); + } catch (Exception e) { + Utils.saveExceptionInDatabase("GitLab basic data scraper", "repository_url", basicData.software(), e); + Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "basic_data_last_error", basicData.software().toString(), "software", scrapedAt, "basic_data_scraped_at"); + } + }); + futures[i] = future; + i++; + } + CompletableFuture.allOf(futures).join(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainCommits.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainCommits.java new file mode 100644 index 0000000..16fcd4a --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainCommits.java @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2025 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2022 - 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdRateLimitException; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +public class MainCommits { + + private static final Logger LOGGER = LoggerFactory.getLogger(MainCommits.class); + + public static void main(String[] args) { + + LOGGER.info("Start scraping commits"); + + long t1 = System.currentTimeMillis(); + + scrapeGitHub(); + scrapeGitLab(); + scrape4tu(); + + long time = System.currentTimeMillis() - t1; + + LOGGER.info("Done scraping commits ({} ms.)", time); + } + + private static void scrapeGitLab() { + PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITLAB); + Collection dataToScrape = softwareInfoRepository.commitDataWithHistory(Config.maxRequestsGitLab()); + CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; + ZonedDateTime scrapedAt = ZonedDateTime.now(); + int i = 0; + for (BasicRepositoryDataWithHistory repositoryDataToScrape : dataToScrape) { + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + String repoUrl = repositoryDataToScrape.url(); + String hostname = URI.create(repoUrl).getHost(); + String apiUrl = "https://" + hostname + "/api"; + String projectPath = repoUrl.replace("https://" + hostname + "/", ""); + if (projectPath.endsWith("/")) projectPath = projectPath.substring(0, projectPath.length() - 1); + + CommitsPerWeek existingCommitsPerWeek = repositoryDataToScrape.commitsPerWeek(); + ZonedDateTime lastCommitHistoryTimestamp = existingCommitsPerWeek.popLatestTimestamp(); + CommitsPerWeek scrapedCommits = new GitlabScraper(apiUrl, projectPath, lastCommitHistoryTimestamp, existingCommitsPerWeek).contributions(); + + BasicRepositoryData basicData = new BasicRepositoryData(repositoryDataToScrape.software(), repositoryDataToScrape.url()); + CommitData updatedData = new CommitData(basicData, scrapedCommits, scrapedAt); + softwareInfoRepository.saveCommitData(updatedData); + } catch (RsdRateLimitException e) { + Utils.saveExceptionInDatabase("GitLab commit scraper", "repository_url", repositoryDataToScrape.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "commit_history_last_error", repositoryDataToScrape.software().toString(), "software", null, null); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase("GitLab commit scraper", "repository_url", repositoryDataToScrape.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "commit_history_last_error", repositoryDataToScrape.software().toString(), "software", scrapedAt, "commit_history_scraped_at"); + } catch (Exception e) { + Utils.saveExceptionInDatabase("GitLab commit scraper", "repository_url", repositoryDataToScrape.software(), e); + Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "commit_history_last_error", repositoryDataToScrape.software().toString(), "software", scrapedAt, "commit_history_scraped_at"); + } + }); + futures[i] = future; + i++; + } + CompletableFuture.allOf(futures).join(); + } + + private static void scrapeGitHub() { + PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITHUB); + Collection dataToScrape = softwareInfoRepository.commitData(Config.maxRequestsGithub()); + CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; + ZonedDateTime scrapedAt = ZonedDateTime.now(); + int i = 0; + for (BasicRepositoryData commitData : dataToScrape) { + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + String repoUrl = commitData.url(); + + Optional githubScraperOptional = GithubScraper.create(repoUrl); + if (githubScraperOptional.isEmpty()) { + Utils.saveErrorMessageInDatabase("Not a valid GitHub URL: " + repoUrl, "repository_url", "commit_history_last_error", commitData.software().toString(), "software", scrapedAt, "commit_history_scraped_at"); + return; + } + + GithubScraper githubScraper = githubScraperOptional.get(); + CommitsPerWeek scrapedCommits = githubScraper.contributions(); + CommitData updatedData = new CommitData(commitData, scrapedCommits, scrapedAt); + softwareInfoRepository.saveCommitData(updatedData); + } catch (RsdRateLimitException e) { + // in case we hit the rate limit, we don't update the scraped_at time, so it gets scraped first next time + Utils.saveExceptionInDatabase("GitHub commit scraper", "repository_url", commitData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "commit_history_last_error", commitData.software().toString(), "software", null, null); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase("GitHub commit scraper", "repository_url", commitData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "commit_history_last_error", commitData.software().toString(), "software", scrapedAt, "commit_history_scraped_at"); + } catch (Exception e) { + Utils.saveExceptionInDatabase("GitHub commit scraper", "repository_url", commitData.software(), e); + Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "commit_history_last_error", commitData.software().toString(), "software", scrapedAt, "commit_history_scraped_at"); + } + }); + futures[i] = future; + i++; + } + CompletableFuture.allOf(futures).join(); + } + + private static void scrape4tu() { + PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.FOURTU); + Collection dataToScrape = softwareInfoRepository.commitData(Config.maxRequestsGithub()); + CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; + ZonedDateTime scrapedAt = ZonedDateTime.now(); + int i = 0; + for (BasicRepositoryData commitData : dataToScrape) { + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + String repoUrl = commitData.url(); + + FourTuGitScraper fourTuGitScraper = new FourTuGitScraper(repoUrl); + CommitsPerWeek scrapedCommits = fourTuGitScraper.contributions(); + CommitData updatedData = new CommitData(commitData, scrapedCommits, scrapedAt); + softwareInfoRepository.saveCommitData(updatedData); + } catch (RsdRateLimitException e) { + // in case we hit the rate limit, we don't update the scraped_at time, so it gets scraped first next time + Utils.saveExceptionInDatabase("4TU commit scraper", "repository_url", commitData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "commit_history_last_error", commitData.software().toString(), "software", null, null); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase("4TU commit scraper", "repository_url", commitData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "commit_history_last_error", commitData.software().toString(), "software", scrapedAt, "commit_history_scraped_at"); + } catch (Exception e) { + Utils.saveExceptionInDatabase("4TU commit scraper", "repository_url", commitData.software(), e); + Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "commit_history_last_error", commitData.software().toString(), "software", scrapedAt, "commit_history_scraped_at"); + } + }); + futures[i] = future; + i++; + } + CompletableFuture.allOf(futures).join(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainContributors.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainContributors.java new file mode 100644 index 0000000..e2dd0dd --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainContributors.java @@ -0,0 +1,157 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdRateLimitException; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +public class MainContributors { + + private static final Logger LOGGER = LoggerFactory.getLogger(MainContributors.class); + + public static void main(String[] args) { + LOGGER.info("Start scraping contributors"); + + long t1 = System.currentTimeMillis(); + + scrapeGitHub(); + scrapeGitLab(); + scrape4tu(); + + long time = System.currentTimeMillis() - t1; + + LOGGER.info("Done scraping contributors ({} ms.)", time); + } + + private static void scrapeGitHub() { + PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITHUB); + Collection dataToScrape = softwareInfoRepository.contributorData(Config.maxRequestsGithub()); + CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; + ZonedDateTime scrapedAt = ZonedDateTime.now(); + int i = 0; + for (BasicRepositoryData contributorData : dataToScrape) { + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + String repoUrl = contributorData.url(); + + Optional githubScraperOptional = GithubScraper.create(repoUrl); + if (githubScraperOptional.isEmpty()) { + Utils.saveErrorMessageInDatabase("Not a valid GitHub URL: " + repoUrl, "repository_url", "contributor_count_last_error", contributorData.software() + .toString(), "software", scrapedAt, "contributor_count_scraped_at"); + return; + } + + GithubScraper githubScraper = githubScraperOptional.get(); + Integer scrapedContributorData = githubScraper.contributorCount(); + ContributorDatabaseData updatedData = new ContributorDatabaseData(new BasicRepositoryData(contributorData.software(), null), scrapedContributorData, scrapedAt); + softwareInfoRepository.saveContributorCount(updatedData); + } catch (RsdRateLimitException e) { + // in case we hit the rate limit, we don't update the scraped_at time, so it gets scraped first next time + Utils.saveExceptionInDatabase("GitHub contributor scraper", "repository_url", contributorData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "contributor_count_last_error", contributorData.software() + .toString(), "software", null, null); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase("GitHub contributor scraper", "repository_url", contributorData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "contributor_count_last_error", contributorData.software() + .toString(), "software", scrapedAt, "contributor_count_scraped_at"); + } catch (Exception e) { + Utils.saveExceptionInDatabase("GitHub contributor scraper", "repository_url", contributorData.software(), e); + Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "contributor_count_last_error", contributorData.software() + .toString(), "software", scrapedAt, "contributor_count_scraped_at"); + } + }); + futures[i] = future; + i++; + } + CompletableFuture.allOf(futures).join(); + } + + private static void scrapeGitLab() { + PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITLAB); + Collection dataToScrape = softwareInfoRepository.contributorData(Config.maxRequestsGithub()); + CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; + ZonedDateTime scrapedAt = ZonedDateTime.now(); + int i = 0; + for (BasicRepositoryData contributorData : dataToScrape) { + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + String repoUrl = contributorData.url(); + String hostname = URI.create(repoUrl).getHost(); + String apiUrl = "https://" + hostname + "/api"; + String projectPath = repoUrl.replace("https://" + hostname + "/", ""); + if (projectPath.endsWith("/")) projectPath = projectPath.substring(0, projectPath.length() - 1); + + Integer scrapedContributorData = new GitlabScraper(apiUrl, projectPath).contributorCount(); + ContributorDatabaseData updatedData = new ContributorDatabaseData(new BasicRepositoryData(contributorData.software(), null), scrapedContributorData, scrapedAt); + softwareInfoRepository.saveContributorCount(updatedData); + } catch (RsdRateLimitException e) { + // in case we hit the rate limit, we don't update the scraped_at time, so it gets scraped first next time + Utils.saveExceptionInDatabase("GitLab contributor scraper", "repository_url", contributorData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "contributor_count_last_error", contributorData.software() + .toString(), "software", null, null); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase("GitLab contributor scraper", "repository_url", contributorData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "contributor_count_last_error", contributorData.software() + .toString(), "software", scrapedAt, "contributor_count_scraped_at"); + } catch (Exception e) { + Utils.saveExceptionInDatabase("GitLab contributor scraper", "repository_url", contributorData.software(), e); + Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "contributor_count_last_error", contributorData.software() + .toString(), "software", scrapedAt, "contributor_count_scraped_at"); + } + }); + futures[i] = future; + i++; + } + CompletableFuture.allOf(futures).join(); + } + + private static void scrape4tu() { + PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.FOURTU); + Collection dataToScrape = softwareInfoRepository.contributorData(Config.maxRequestsGithub()); + CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; + ZonedDateTime scrapedAt = ZonedDateTime.now(); + int i = 0; + for (BasicRepositoryData contributorData : dataToScrape) { + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + String repoUrl = contributorData.url(); + + FourTuGitScraper fourTuGitScraper = new FourTuGitScraper(repoUrl); + + Integer scrapedContributorData = fourTuGitScraper.contributorCount(); + ContributorDatabaseData updatedData = new ContributorDatabaseData(new BasicRepositoryData(contributorData.software(), null), scrapedContributorData, scrapedAt); + softwareInfoRepository.saveContributorCount(updatedData); + } catch (RsdRateLimitException e) { + // in case we hit the rate limit, we don't update the scraped_at time, so it gets scraped first next time + Utils.saveExceptionInDatabase("4TU contributor scraper", "repository_url", contributorData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "contributor_count_last_error", contributorData.software() + .toString(), "software", null, null); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase("4TU contributor scraper", "repository_url", contributorData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "contributor_count_last_error", contributorData.software() + .toString(), "software", scrapedAt, "contributor_count_scraped_at"); + } catch (Exception e) { + Utils.saveExceptionInDatabase("4TU contributor scraper", "repository_url", contributorData.software(), e); + Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "contributor_count_last_error", contributorData.software() + .toString(), "software", scrapedAt, "contributor_count_scraped_at"); + } + }); + futures[i] = future; + i++; + } + CompletableFuture.allOf(futures).join(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainProgrammingLanguages.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainProgrammingLanguages.java new file mode 100644 index 0000000..7e7752d --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainProgrammingLanguages.java @@ -0,0 +1,154 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdRateLimitException; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +public class MainProgrammingLanguages { + + private static final Logger LOGGER = LoggerFactory.getLogger(MainProgrammingLanguages.class); + + public static void main(String[] args) { + + LOGGER.info("Start scraping programming languages"); + + long t1 = System.currentTimeMillis(); + + scrapeGithub(); + scrapeGitLab(); + scrape4tu(); + + long time = System.currentTimeMillis() - t1; + + LOGGER.info("Done scraping programming languages ({} ms.)", time); + } + + private static void scrapeGitLab() { + PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITLAB); + Collection dataToScrape = softwareInfoRepository.languagesData(Config.maxRequestsGitLab()); + CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; + ZonedDateTime scrapedAt = ZonedDateTime.now(); + int i = 0; + for (BasicRepositoryData programmingLanguageData : dataToScrape) { + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + String repoUrl = programmingLanguageData.url(); + String hostname = URI.create(repoUrl).getHost(); + String apiUrl = "https://" + hostname + "/api"; + String projectPath = repoUrl.replace("https://" + hostname + "/", ""); + if (projectPath.endsWith("/")) projectPath = projectPath.substring(0, projectPath.length() - 1); + + String scrapedLanguages = new GitlabScraper(apiUrl, projectPath).languages(); + LanguagesData updatedData = new LanguagesData(new BasicRepositoryData(programmingLanguageData.software(), null), scrapedLanguages, scrapedAt); + softwareInfoRepository.saveLanguagesData(updatedData); + } catch (RsdRateLimitException e) { + Utils.saveExceptionInDatabase("GitLab programming languages scraper", "repository_url", programmingLanguageData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "languages_last_error", programmingLanguageData.software() + .toString(), "software", null, null); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase("GitLab programming languages scraper", "repository_url", programmingLanguageData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "languages_last_error", programmingLanguageData.software() + .toString(), "software", scrapedAt, "languages_scraped_at"); + } catch (Exception e) { + Utils.saveExceptionInDatabase("GitLab programming languages scraper", "repository_url", programmingLanguageData.software(), e); + Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "languages_last_error", programmingLanguageData.software() + .toString(), "software", scrapedAt, "languages_scraped_at"); + } + }); + futures[i] = future; + i++; + } + CompletableFuture.allOf(futures).join(); + } + + private static void scrapeGithub() { + PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITHUB); + Collection dataToScrape = softwareInfoRepository.languagesData(Config.maxRequestsGithub()); + CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; + ZonedDateTime scrapedAt = ZonedDateTime.now(); + int i = 0; + for (BasicRepositoryData programmingLanguageData : dataToScrape) { + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + String repoUrl = programmingLanguageData.url(); + + Optional githubScraperOptional = GithubScraper.create(repoUrl); + if (githubScraperOptional.isEmpty()) { + Utils.saveErrorMessageInDatabase("Not a valid GitHub URL: " + repoUrl, "repository_url", "languages_last_error", programmingLanguageData.software() + .toString(), "software", scrapedAt, "languages_scraped_at"); + return; + } + + GithubScraper githubScraper = githubScraperOptional.get(); + String scrapedLanguages = githubScraper.languages(); + LanguagesData updatedData = new LanguagesData(new BasicRepositoryData(programmingLanguageData.software(), null), scrapedLanguages, scrapedAt); + softwareInfoRepository.saveLanguagesData(updatedData); + } catch (RsdRateLimitException e) { + Utils.saveExceptionInDatabase("GitHub programming languages scraper", "repository_url", programmingLanguageData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "languages_last_error", programmingLanguageData.software() + .toString(), "software", null, null); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase("GitHub programming languages scraper", "repository_url", programmingLanguageData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "languages_last_error", programmingLanguageData.software() + .toString(), "software", scrapedAt, "languages_scraped_at"); + } catch (Exception e) { + Utils.saveExceptionInDatabase("GitHub programming languages scraper", "repository_url", programmingLanguageData.software(), e); + Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "languages_last_error", programmingLanguageData.software() + .toString(), "software", scrapedAt, "languages_scraped_at"); + } + }); + futures[i] = future; + i++; + } + CompletableFuture.allOf(futures).join(); + } + + private static void scrape4tu() { + PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.FOURTU); + Collection dataToScrape = softwareInfoRepository.languagesData(10); + CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; + ZonedDateTime scrapedAt = ZonedDateTime.now(); + int i = 0; + for (BasicRepositoryData programmingLanguageData : dataToScrape) { + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + String repoUrl = programmingLanguageData.url(); + + FourTuGitScraper fourTuGitScraper = new FourTuGitScraper(repoUrl); + String scrapedLanguages = fourTuGitScraper.languages(); + LanguagesData updatedData = new LanguagesData(new BasicRepositoryData(programmingLanguageData.software(), null), scrapedLanguages, scrapedAt); + softwareInfoRepository.saveLanguagesData(updatedData); + } catch (RsdRateLimitException e) { + Utils.saveExceptionInDatabase("4TU programming languages scraper", "repository_url", programmingLanguageData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "languages_last_error", programmingLanguageData.software() + .toString(), "software", null, null); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase("4TU programming languages scraper", "repository_url", programmingLanguageData.software(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "languages_last_error", programmingLanguageData.software() + .toString(), "software", scrapedAt, "languages_scraped_at"); + } catch (Exception e) { + Utils.saveExceptionInDatabase("4TU programming languages scraper", "repository_url", programmingLanguageData.software(), e); + Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "languages_last_error", programmingLanguageData.software() + .toString(), "software", scrapedAt, "languages_scraped_at"); + } + }); + futures[i] = future; + i++; + } + CompletableFuture.allOf(futures).join(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnector.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnector.java new file mode 100644 index 0000000..cdb626d --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnector.java @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2025 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.Utils; + +import java.time.Instant; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Objects; +import java.util.UUID; + +public class PostgrestConnector { + + private final String backendUrl; + private final String filter; + + public PostgrestConnector(String backendUrl, CodePlatformProvider codePlatform) { + this.backendUrl = Objects.requireNonNull(backendUrl); + Objects.requireNonNull(codePlatform); + String codePlatformQueryValue = switch (codePlatform) { + case FOURTU -> "4tu"; + default -> codePlatform.name().toLowerCase(); + }; + this.filter = "scraping_disabled_reason=is.null&code_platform=eq." + codePlatformQueryValue; + } + + /** + * Fetch programming languages data from PostgREST + * + * @param limit The number of rows requested from PostgREST + * @return The data corresponding to the git repositories of which the programming languages data were scraped the longest time ago + */ + public Collection languagesData(int limit) { + String data = Utils.getAsAdmin(backendUrl + "?" + filter + "&select=software,url&order=languages_scraped_at.asc.nullsfirst&limit=" + limit + "&" + Utils.atLeastOneHourAgoFilter("languages_scraped_at")); + return parseBasicJsonData(data); + } + + /** + * Fetch commit data from PostgREST + * + * @param limit The number of rows requested from PostgREST + * @return The data corresponding to the git repositories of which the commit data were scraped the longest time ago + */ + public Collection commitData(int limit) { + String data = Utils.getAsAdmin(backendUrl + "?" + filter + "&select=software,url&order=commit_history_scraped_at.asc.nullsfirst&limit=" + limit + "&" + Utils.atLeastOneHourAgoFilter("commit_history_scraped_at")); + return parseBasicJsonData(data); + } + + public Collection commitDataWithHistory(int limit) { + String data = Utils.getAsAdmin(backendUrl + "?" + filter + "&select=software,url,commit_history_scraped_at,commit_history&order=commit_history_scraped_at.asc.nullsfirst&limit=" + limit + "&" + Utils.atLeastOneHourAgoFilter("commit_history_scraped_at")); + return parseBasicJsonDataWithHistory(data); + } + + /** + * Fetch basic data from PostgREST + * + * @param limit The number of rows requested from PostgREST + * @return The data corresponding to the git repositories of which the basic data were scraped the longest time ago + */ + public Collection statsData(int limit) { + String data = Utils.getAsAdmin(backendUrl + "?" + filter + "&select=software,url&order=basic_data_scraped_at.asc.nullsfirst&limit=" + limit + "&" + Utils.atLeastOneHourAgoFilter("basic_data_scraped_at")); + return parseBasicJsonData(data); + } + + public Collection contributorData(int limit) { + String data = Utils.getAsAdmin(backendUrl + "?" + filter + "&select=software,url&order=contributor_count_scraped_at.asc.nullsfirst&limit=" + limit + "&" + Utils.atLeastOneHourAgoFilter("contributor_count_scraped_at")); + return parseBasicJsonData(data); + } + + static Collection parseBasicJsonDataWithHistory(String data) { + JsonArray dataInArray = JsonParser.parseString(data).getAsJsonArray(); + Collection result = new ArrayList<>(); + for (JsonElement element : dataInArray) { + JsonObject jsonObject = element.getAsJsonObject(); + String softwareUuid = jsonObject.getAsJsonPrimitive("software").getAsString(); + UUID software = UUID.fromString(softwareUuid); + String url = jsonObject.getAsJsonPrimitive("url").getAsString(); + String commitHistoryScrapedAtString = Utils.stringOrNull(jsonObject.get("commit_history_scraped_at")); + ZonedDateTime commitHistoryScrapedAt = commitHistoryScrapedAtString == null ? null : ZonedDateTime.parse(commitHistoryScrapedAtString); + CommitsPerWeek commitsPerWeek = new CommitsPerWeek(); + if (!jsonObject.get("commit_history").isJsonNull()) { + JsonObject commitHistoryJsonObject = jsonObject.getAsJsonObject("commit_history"); + for (String key : commitHistoryJsonObject.keySet()) { + commitsPerWeek.addCommits(Instant.ofEpochSecond(Long.parseLong(key)), commitHistoryJsonObject.getAsJsonPrimitive(key).getAsLong()); + } + } + result.add(new BasicRepositoryDataWithHistory(software, url, commitHistoryScrapedAt, commitsPerWeek)); + } + return result; + } + + static Collection parseBasicJsonData(String data) { + JsonArray dataInArray = JsonParser.parseString(data).getAsJsonArray(); + Collection result = new ArrayList<>(); + for (JsonElement element : dataInArray) { + JsonObject jsonObject = element.getAsJsonObject(); + String softwareUuid = jsonObject.getAsJsonPrimitive("software").getAsString(); + UUID software = UUID.fromString(softwareUuid); + String url = jsonObject.getAsJsonPrimitive("url").getAsString(); + + result.add(new BasicRepositoryData(software, url)); + } + return result; + } + + public void saveLanguagesData(LanguagesData languagesData) { + String json = String.format("{\"languages_last_error\": null, \"languages\": %s, \"languages_scraped_at\": \"%s\"}", languagesData.languages(), languagesData.languagesScrapedAt() + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + Utils.patchAsAdmin(backendUrl + "?software=eq." + languagesData.basicData().software().toString(), json); + } + + public void saveCommitData(CommitData commitData) { + String json; + if (commitData.commitHistory() == null) { + json = String.format("{\"commit_history_last_error\": null, \"commit_history_scraped_at\": \"%s\"}", commitData.commitHistoryScrapedAt() + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + } else { + commitData.commitHistory().addMissingZeros(); + json = String.format("{\"commit_history_last_error\": null, \"commit_history\": %s, \"commit_history_scraped_at\": \"%s\"}", commitData.commitHistory() + .toJson(), commitData.commitHistoryScrapedAt().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + } + Utils.patchAsAdmin(backendUrl + "?software=eq." + commitData.basicData().software().toString(), json); + } + + public void saveBasicData(BasicGitDatabaseData basicData) { + JsonObject jsonObject = new JsonObject(); + jsonObject.add("basic_data_last_error", JsonNull.INSTANCE); + jsonObject.addProperty("basic_data_scraped_at", basicData.dataScrapedAt() + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + jsonObject.addProperty("archived", basicData.statsData().archived()); + jsonObject.addProperty("license", basicData.statsData().license()); + jsonObject.addProperty("star_count", basicData.statsData().starCount()); + jsonObject.addProperty("fork_count", basicData.statsData().forkCount()); + jsonObject.addProperty("open_issue_count", basicData.statsData().openIssueCount()); + + Utils.patchAsAdmin(backendUrl + "?software=eq." + basicData.basicData() + .software() + .toString(), jsonObject.toString()); + } + + public void saveContributorCount(ContributorDatabaseData contributorData) { + JsonObject jsonObject = new JsonObject(); + jsonObject.add("contributor_count_last_error", JsonNull.INSTANCE); + jsonObject.addProperty("contributor_count_scraped_at", contributorData.dataScrapedAt() + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + if (contributorData.contributorCount() != null) { + jsonObject.addProperty("contributor_count", contributorData.contributorCount()); + } + + Utils.patchAsAdmin(backendUrl + "?software=eq." + contributorData.basicData() + .software() + .toString(), jsonObject.toString()); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepository.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepository.java new file mode 100644 index 0000000..96df10a --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepository.java @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.license; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.reflect.TypeToken; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; + +import java.io.IOException; +import java.net.http.HttpResponse; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class GitHubSpdxLicenseRepository { + + private static final Gson gson = new Gson(); + + private GitHubSpdxLicenseRepository() { + } + + public static Map getLicensesByIdMap() throws IOException, InterruptedException, RsdResponseException { + String url = "https://raw.githubusercontent.com/spdx/license-list-data/refs/heads/main/json/licenses.json"; + + HttpResponse response = Utils.getAsHttpResponse(url); + if (response.statusCode() != 200) { + throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response while getting SPDX licenses"); + } + + String json = response.body(); + return parseLicensesJson(json); + } + + static Map parseLicensesJson(String json) { + JsonObject root = JsonParser.parseString(json).getAsJsonObject(); + JsonArray licensesJsonArray = root.getAsJsonArray("licenses"); + TypeToken> spdxListTypeToken = new TypeToken<>() { + }; + List licenses = gson.fromJson(licensesJsonArray, spdxListTypeToken); + + return licenses + .stream() + .collect(Collectors.toMap(SpdxLicense::licenseId, Function.identity())); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/SpdxLicense.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/SpdxLicense.java new file mode 100644 index 0000000..7300426 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/SpdxLicense.java @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.license; + +public record SpdxLicense( + String licenseId, + String reference, + String name +) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/MainNassa.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/MainNassa.java new file mode 100644 index 0000000..28378f2 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/MainNassa.java @@ -0,0 +1,166 @@ +// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.nassa; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.BibtexParser; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; +import nl.esciencecenter.rsd.scraper.doi.ExternalMentionRecord; +import nl.esciencecenter.rsd.scraper.license.GitHubSpdxLicenseRepository; +import nl.esciencecenter.rsd.scraper.license.SpdxLicense; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.StringReader; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +public class MainNassa { + + private static final Logger LOGGER = LoggerFactory.getLogger(MainNassa.class); + private static final HttpClient HTTP_CLIENT = HttpClient.newHttpClient(); + private static final Pattern NASSA_MODULE_PATTERN = Pattern.compile("^\\d\\d\\d\\d-\\w+-\\d\\d\\d$"); + private static final Set DENY_LIST_MODULES = Set.of("1870-Schliemann-001"); + private static final String NASSA_SERVICE_NAME = "NASSA scraper"; + + // https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28 + public static void main(String[] args) { + if (!Config.isNassaScraperEnabled()) { + return; + } + + URI rootContentUri = URI.create("https://api.github.com/repos/Archaeology-ABM/NASSA-modules/contents/"); + HttpRequest httpRequest = HttpRequest + .newBuilder(rootContentUri) + .GET() + .build(); + + HttpResponse httpResponse; + try { + httpResponse = HTTP_CLIENT.send(httpRequest, HttpResponse.BodyHandlers.ofString()); + } catch (Exception e) { + Utils.saveExceptionInDatabase(NASSA_SERVICE_NAME, null, null, e); + return; + } + if (httpResponse.statusCode() != 200) { + String message = "Got a non-200 response from the GitHub API querying then contents of the NASSA repository"; + LOGGER.error(message); + RsdResponseException exceptionToSave = new RsdResponseException(httpResponse.statusCode(), rootContentUri, httpResponse.body(), message); + Utils.saveExceptionInDatabase(NASSA_SERVICE_NAME, null, null, exceptionToSave); + return; + } + + JsonArray jsonArray = JsonParser.parseString(httpResponse.body()).getAsJsonArray(); + Collection modules = new ArrayList<>(); + for (JsonElement jsonElement : jsonArray) { + String fileOrDirName = jsonElement.getAsJsonObject().getAsJsonPrimitive("name").getAsString(); + String htmlUrl = jsonElement.getAsJsonObject().getAsJsonPrimitive("html_url").getAsString(); + if (!DENY_LIST_MODULES.contains(fileOrDirName) && NASSA_MODULE_PATTERN.asPredicate().test(fileOrDirName)) { + modules.add(new String[]{fileOrDirName, htmlUrl}); + } + } + + List modulesToSave = new ArrayList<>(modules.size()); + List> mentionsOfModules = new ArrayList<>(modules.size()); + for (String[] moduleInfo : modules) { + try { + // e.g. https://raw.githubusercontent.com/Archaeology-ABM/NASSA-modules/main/1870-Schliemann-001/NASSA.yml + URI ymlUri = URI.create("https://raw.githubusercontent.com/Archaeology-ABM/NASSA-modules/main/%s/NASSA.yml".formatted(moduleInfo[0])); + httpRequest = HttpRequest + .newBuilder(ymlUri) + .GET() + .build(); + + httpResponse = HTTP_CLIENT.send(httpRequest, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + StringReader ymlStringReader = new StringReader(httpResponse.body()); + + String longDescription = null; + try { + URI readmeUri = URI.create("https://raw.githubusercontent.com/Archaeology-ABM/NASSA-modules/main/%s/README.md".formatted(moduleInfo[0])); + longDescription = NassaReadmeParser.downloadAndReturnDescriptionFromReadme(readmeUri, HTTP_CLIENT); + } catch (Exception e) { + Utils.saveExceptionInDatabase(NASSA_SERVICE_NAME, null, null, e); + if (e instanceof InterruptedException) { + return; + } + } + + // e.g. https://raw.githubusercontent.com/Archaeology-ABM/NASSA-modules/main/1870-Schliemann-001/references.bib + URI bibUri = URI.create("https://raw.githubusercontent.com/Archaeology-ABM/NASSA-modules/main/%s/references.bib".formatted(moduleInfo[0])); + httpRequest = HttpRequest + .newBuilder(bibUri) + .GET() + .build(); + httpResponse = HTTP_CLIENT.send(httpRequest, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + StringReader bibStringReader = new StringReader(httpResponse.body()); + Map mentions; + try { + mentions = BibtexParser.parse(bibStringReader); + } catch (Exception e) { + String message = "Exception parsing the Bibtex file of the following module: " + Arrays.toString(moduleInfo); + Exception exceptionToSave = new Exception(message, e); + Utils.saveExceptionInDatabase(NASSA_SERVICE_NAME, null, null, exceptionToSave); + mentions = Map.of(); + } + mentionsOfModules.add(mentions); + + LOGGER.info("parsing module: {} at URL {}", moduleInfo[0], ymlUri); + NassaSoftwareEntry softwareEntry = NassaSoftwareEntry.fromYaml(ymlStringReader, moduleInfo[1], longDescription); + modulesToSave.add(softwareEntry); + LOGGER.info("done parsing module: {}", moduleInfo[0]); + } catch (InterruptedException e) { + String message = "InterruptedException handling the following module: " + Arrays.toString(moduleInfo); + Exception exceptionToSave = new Exception(message, e); + Utils.saveExceptionInDatabase(NASSA_SERVICE_NAME, null, null, exceptionToSave); + return; + } catch (Exception e) { + String message = "Exception handling the following module: " + Arrays.toString(moduleInfo); + Exception exceptionToSave = new Exception(message, e); + Utils.saveExceptionInDatabase(NASSA_SERVICE_NAME, null, null, exceptionToSave); + } + } + + Map licenseMap = Collections.emptyMap(); + try { + licenseMap = GitHubSpdxLicenseRepository.getLicensesByIdMap(); + } catch (Exception e) { + LOGGER.error("Error when getting SPDX licence map from GitHub: {}", e.getMessage()); + Utils.saveExceptionInDatabase(NASSA_SERVICE_NAME, null, null, e); + if (e instanceof InterruptedException) { + return; + } + } + + for (int i = 0; i < modulesToSave.size(); i++) { + NassaSoftwareEntry nassaSoftwareEntry = modulesToSave.get(i); + Map mentions = mentionsOfModules.get(i); + LOGGER.info("saving module to database: {}", nassaSoftwareEntry.id); + try { + Utils.postAsAdmin(Config.backendBaseUrl() + "/rpc/nassa_import", nassaSoftwareEntry.toRsdJson(mentions, licenseMap)); + } catch (Exception e) { + String message = "Exception handling the following module: " + nassaSoftwareEntry.id; + Exception exceptionToSave = new Exception(message, e); + Utils.saveExceptionInDatabase(NASSA_SERVICE_NAME, null, null, exceptionToSave); + } + LOGGER.info("done saving module to database: {}", nassaSoftwareEntry.id); + } + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaReadmeParser.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaReadmeParser.java new file mode 100644 index 0000000..9b8581b --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaReadmeParser.java @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.nassa; + +import nl.esciencecenter.rsd.scraper.RsdResponseException; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.Objects; +import java.util.regex.Pattern; + +public class NassaReadmeParser { + + static Pattern furtherInfoPattern = Pattern.compile("## Further information\\s+"); + + static String getLongDescriptionFromReadme(String content) throws RsdNassaParseException { + Objects.requireNonNull(content); + + String[] split = furtherInfoPattern.split(content); + if (split.length == 2) { + return split[1]; + } + + throw new RsdNassaParseException("Couldn't split description into two parts around expected header, resulting array has size %d instead of 2".formatted(split.length)); + } + + public static String downloadAndReturnDescriptionFromReadme(URI readmeUri, HttpClient client) throws IOException, InterruptedException, RsdResponseException, RsdNassaParseException { + HttpRequest request = HttpRequest.newBuilder(readmeUri).build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() >= 300) { + throw new RsdResponseException(response.statusCode(), readmeUri, response.body(), "Unexpected response getting NASSA README"); + } + + return getLongDescriptionFromReadme(response.body()); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntry.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntry.java new file mode 100644 index 0000000..ee12d39 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntry.java @@ -0,0 +1,369 @@ +// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.nassa; + +import com.google.gson.JsonArray; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import nl.esciencecenter.rsd.scraper.doi.ExternalMentionRecord; +import nl.esciencecenter.rsd.scraper.doi.PostgrestMentionRepository; +import nl.esciencecenter.rsd.scraper.license.SpdxLicense; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; + +import java.io.Reader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class NassaSoftwareEntry { + + private static final Logger LOGGER = LoggerFactory.getLogger(NassaSoftwareEntry.class); + + final String gitHtmlUrl; + final String id; + final String nassaVersion; + final String moduleType; + final String title; + final String moduleVersion; + final List contributors; + final String license; + final Date lastUpdateDate; + final String shortDescription; + final String longDescription; + final List relatedModules; + final Map> references; + final Map> domainKeywords; + final List modellingKeywords; + final List programmingKeywords; + final List> implementations; + final String docsDir; + final List> inputs; + final List> outputs; + + private NassaSoftwareEntry( + String gitHtmlUrl, + String id, + String nassaVersion, + String moduleType, + String title, + String moduleVersion, + List contributors, + String license, + Date lastUpdateDate, + String shortDescription, + String longDescription, + List relatedModules, + Map> references, + Map> domainKeywords, + List modellingKeywords, + List programmingKeywords, + List> implementations, + String docsDir, + List> inputs, + List> outputs + ) { + this.gitHtmlUrl = gitHtmlUrl; + this.id = id; + this.nassaVersion = nassaVersion; + this.moduleType = moduleType; + this.title = title; + this.moduleVersion = moduleVersion; + this.contributors = contributors; + this.license = license; + this.lastUpdateDate = lastUpdateDate; + this.shortDescription = shortDescription; + this.longDescription = longDescription; + this.relatedModules = relatedModules; + this.references = references; + this.domainKeywords = Objects.requireNonNull(domainKeywords); + this.modellingKeywords = modellingKeywords; + this.programmingKeywords = programmingKeywords; + this.implementations = implementations; + this.docsDir = docsDir; + this.inputs = inputs; + this.outputs = outputs; + } + + public static NassaSoftwareEntry fromYaml(Reader yamlSource, String gitHtmlUrl, String longDescription) { + Yaml yaml = new Yaml(); + Map map = yaml.load(yamlSource); + + String id = (String) map.get("id"); + String nassaVersion = (String) map.get("nassaVersion"); + String moduleType = (String) map.get("moduleType"); + String title = (String) map.get("title"); + String moduleVersion = (String) map.get("moduleVersion"); + @SuppressWarnings("unchecked") + List> contributorsUnparsed = (List>) map.get("contributors"); + List contributors = new ArrayList<>(contributorsUnparsed.size()); + int position = 1; + for (Map contributorMap : contributorsUnparsed) { + contributors.add(NassaContributor.fromParsedYaml(contributorMap, position)); + position++; + } + String license = (String) map.get("license"); + Date lastUpdateDate = (Date) map.get("lastUpdateDate"); + String shortDescription = (String) map.get("description"); + @SuppressWarnings("unchecked") + List relatedModules = (List) map.get("relatedModules"); + @SuppressWarnings("unchecked") + Map> references = (Map>) map.get("references"); + @SuppressWarnings("unchecked") + Map> domainKeywords = (Map>) map.get("domainKeywords"); + if (domainKeywords == null) { + domainKeywords = Map.of(); + } + @SuppressWarnings("unchecked") + List modellingKeywords = (List) map.get("modellingKeywords"); + @SuppressWarnings("unchecked") + List programmingKeywords = (List) map.get("programmingKeywords"); + @SuppressWarnings("unchecked") + List> implementations = (List>) map.get("implementations"); + String docsDir = (String) map.get("docsDir"); + @SuppressWarnings("unchecked") + List> inputs = (List>) map.get("inputs"); + @SuppressWarnings("unchecked") + List> outputs = (List>) map.get("outputs"); + + return new NassaSoftwareEntry( + gitHtmlUrl, + id, + nassaVersion, + moduleType, + title, + moduleVersion, + contributors, + license, + lastUpdateDate, + shortDescription, + longDescription, + relatedModules, + references, + domainKeywords, + modellingKeywords, + programmingKeywords, + implementations, + docsDir, + inputs, + outputs + ); + } + + public String toRsdJson(Map mentions, Map licenseMap) { + JsonObject root = new JsonObject(); + root.addProperty("slug", id.toLowerCase()); + root.addProperty("brand_name", title); + root.addProperty("description", descriptionWithInputsAndOutputs()); + root.addProperty("short_statement", shortDescription.length() > 300 ? shortDescription.substring(0, 297) + "..." : shortDescription); + root.addProperty("get_started_url", gitHtmlUrl + "/" + docsDir); + root.addProperty("repository_url", gitHtmlUrl); + + if (license != null) { + SpdxLicense spdxLicense = licenseMap.get(license); + if (spdxLicense != null) { + root.addProperty("license_value", spdxLicense.licenseId()); + root.addProperty("license_name", spdxLicense.name()); + root.addProperty("license_url", spdxLicense.reference()); + root.addProperty("license_open_source", true); + } else { + LOGGER.warn("Unexpected SPDX license: {}", license); + // we need to add this so we can call the nassa_import RPC with all parameters + root.add("license_value", JsonNull.INSTANCE); + root.add("license_name", JsonNull.INSTANCE); + root.add("license_url", JsonNull.INSTANCE); + root.add("license_open_source", JsonNull.INSTANCE); + } + } + + JsonArray relatedModulesJson = new JsonArray(); + if (relatedModules != null) { + relatedModules.forEach(id -> relatedModulesJson.add(id.toLowerCase())); + } + root.add("related_modules", relatedModulesJson); + + JsonArray familyNamesArray = new JsonArray(contributors.size()); + JsonArray givenNamesArray = new JsonArray(contributors.size()); + JsonArray roleArray = new JsonArray(contributors.size()); + JsonArray orcidArray = new JsonArray(contributors.size()); + JsonArray positionArray = new JsonArray(contributors.size()); + for (NassaContributor contributor : contributors) { + familyNamesArray.add(contributor.familyName); + givenNamesArray.add(contributor.givenName); + roleArray.add(contributor.roles); + orcidArray.add(contributor.orcid); + positionArray.add(contributor.position); + } + + root.add("family_names_array", familyNamesArray); + root.add("given_names_array", givenNamesArray); + root.add("role_array", roleArray); + root.add("orcid_array", orcidArray); + root.add("position_array", positionArray); + + JsonObject categoriesJson = new JsonObject(); + + JsonArray modellingKeywordsJson = new JsonArray(modellingKeywords.size()); + modellingKeywords.forEach(modellingKeywordsJson::add); + categoriesJson.add("Modelling", modellingKeywordsJson); + + JsonArray programmingKeywordsJson = new JsonArray(programmingKeywords.size()); + programmingKeywords.forEach(programmingKeywordsJson::add); + categoriesJson.add("Programming", programmingKeywordsJson); + + List regionsKeywords = domainKeywords.get("regions"); + if (regionsKeywords != null) { + JsonArray regionsKeywordsJson = new JsonArray(regionsKeywords.size()); + regionsKeywords.forEach(regionsKeywordsJson::add); + categoriesJson.add("Region", regionsKeywordsJson); + } + + List periodsKeywords = domainKeywords.getOrDefault("periods", List.of()); + if (periodsKeywords != null) { + JsonArray periodsKeywordsJson = new JsonArray(periodsKeywords.size()); + periodsKeywords.forEach(periodsKeywordsJson::add); + categoriesJson.add("Period", periodsKeywordsJson); + } + + List subjectsKeywords = domainKeywords.getOrDefault("subjects", List.of()); + if (subjectsKeywords != null) { + JsonArray subjectsKeywordsJson = new JsonArray(subjectsKeywords.size()); + subjectsKeywords.forEach(subjectsKeywordsJson::add); + categoriesJson.add("Subject", subjectsKeywordsJson); + } + + if (implementations != null) { + JsonArray languagesJson = new JsonArray(); + for (Map implementation : implementations) { + if (implementation != null && implementation.get("language") != null && implementation.get("language") instanceof String lang) { + languagesJson.add(lang); + } + } + categoriesJson.add("Language", languagesJson); + } + + JsonArray moduleTypeJson = new JsonArray(); + moduleTypeJson.add(moduleType); + categoriesJson.add("Module type", moduleTypeJson); + + root.add("categories", categoriesJson); + + Collection regularMentions = new ArrayList<>(); + if (references != null && references.get("moduleReferences") != null) { + references.get("moduleReferences").forEach(citekey -> { + if (mentions.get(citekey) != null) { + regularMentions.add(mentions.get(citekey)); + } + }); + } + + if (references != null && references.get("useExampleReferences") != null) { + references.get("useExampleReferences").forEach(citekey -> { + if (mentions.get(citekey) != null) { + regularMentions.add(mentions.get(citekey)); + } + }); + } + root.add("regular_mentions", PostgrestMentionRepository.toRsdJsonArray(regularMentions)); + + return root.toString(); + } + + private String descriptionWithInputsAndOutputs() { + StringBuilder descriptionBuilder = new StringBuilder(); + if (longDescription != null) { + descriptionBuilder.append(longDescription); + descriptionBuilder.append('\n'); + descriptionBuilder.append('\n'); + } + + if (inputs != null && !inputs.isEmpty()) { + descriptionBuilder.append('\n'); + descriptionBuilder.append("## Inputs\n"); + descriptionBuilder.append('\n'); + descriptionBuilder.append("|**Name**|**Type**|**Description**|\n"); + descriptionBuilder.append("|-|-|-|\n"); + + for (Map input : inputs) { + String name = input.getOrDefault("name", ""); + String type = input.getOrDefault("type", ""); + String description = input.getOrDefault("description", ""); + descriptionBuilder.append("|%s|%s|%s|\n".formatted(name, type, description)); + } + + descriptionBuilder.append("||||\n"); + } + + if (outputs != null && !outputs.isEmpty()) { + descriptionBuilder.append('\n'); + descriptionBuilder.append("## Outputs\n"); + descriptionBuilder.append('\n'); + descriptionBuilder.append("|**Name**|**Type**|**Description**|\n"); + descriptionBuilder.append("|-|-|-|\n"); + + for (Map output : outputs) { + String name = output.getOrDefault("name", ""); + String type = output.getOrDefault("type", ""); + String description = output.getOrDefault("description", ""); + descriptionBuilder.append("|%s|%s|%s|\n".formatted(name, type, description)); + } + + descriptionBuilder.append("||||\n"); + } + + return descriptionBuilder.toString(); + } +} + +class NassaContributor { + final String givenName; + final String familyName; + final String emailAddress; + final String roles; + final String orcid; + final int position; + + private NassaContributor( + String givenName, + String familyName, + String emailAddress, + String roles, + String orcid, + int position + ) { + this.givenName = givenName; + this.familyName = familyName; + this.emailAddress = emailAddress; + this.roles = roles; + this.orcid = orcid; + this.position = position; + } + + static NassaContributor fromParsedYaml(Map map, int position) { + String name = (String) map.get("name"); + String[] nameSplit = name.split(", "); + String givenName = nameSplit.length >= 2 ? nameSplit[1] : ""; + String familyName = nameSplit[0]; + @SuppressWarnings("unchecked") + List roleList = (List) map.get("roles"); + String roles = String.join(", ", roleList); + String emailAddress = (String) map.get("email"); + String orcid = (String) map.get("orcid"); + + return new NassaContributor( + givenName, + familyName, + emailAddress, + roles, + orcid, + position + ); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/RsdNassaParseException.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/RsdNassaParseException.java new file mode 100644 index 0000000..7657045 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/RsdNassaParseException.java @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.nassa; + +public class RsdNassaParseException extends Exception { + + public RsdNassaParseException(String message) { + super(message); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/BasicPackageManagerData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/BasicPackageManagerData.java new file mode 100644 index 0000000..9e9f680 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/BasicPackageManagerData.java @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2023 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager; + +import java.util.UUID; + +public record BasicPackageManagerData( + UUID id, + String url, + PackageManagerType type +) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/MainPackageManager.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/MainPackageManager.java new file mode 100644 index 0000000..8497d5a --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/MainPackageManager.java @@ -0,0 +1,154 @@ +// SPDX-FileCopyrightText: 2023 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2025 Paula Stock (GFZ) +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager; + +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdRateLimitException; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; +import nl.esciencecenter.rsd.scraper.package_manager.scrapers.AnacondaScraper; +import nl.esciencecenter.rsd.scraper.package_manager.scrapers.CranScraper; +import nl.esciencecenter.rsd.scraper.package_manager.scrapers.CratesScraper; +import nl.esciencecenter.rsd.scraper.package_manager.scrapers.DockerHubScraper; +import nl.esciencecenter.rsd.scraper.package_manager.scrapers.FourTuScraper; +import nl.esciencecenter.rsd.scraper.package_manager.scrapers.GoScraper; +import nl.esciencecenter.rsd.scraper.package_manager.scrapers.MavenScraper; +import nl.esciencecenter.rsd.scraper.package_manager.scrapers.NpmScraper; +import nl.esciencecenter.rsd.scraper.package_manager.scrapers.PackageManagerScraper; +import nl.esciencecenter.rsd.scraper.package_manager.scrapers.PixiScraper; +import nl.esciencecenter.rsd.scraper.package_manager.scrapers.PypiScraper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +public class MainPackageManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(MainPackageManager.class); + + public static void main(String[] args) { + LOGGER.info("Start scraping package manager data"); + + long t1 = System.currentTimeMillis(); + + PostgrestConnector postgrestConnector = new PostgrestConnector(Config.backendBaseUrl() + "/package_manager"); + Collection downloadsToScrape = postgrestConnector.oldestDownloadCounts(10); + Collection revDepsToScrape = postgrestConnector.oldestReverseDependencyCounts(10); + Collection> tasks = new ArrayList<>(); + ZonedDateTime scrapedAt = ZonedDateTime.now(); + for (BasicPackageManagerData basicData : downloadsToScrape) { + Callable task = () -> { + scrapeDownloads(basicData, postgrestConnector, scrapedAt); + return null; + }; + tasks.add(task); + } + for (BasicPackageManagerData basicData : revDepsToScrape) { + Callable task = () -> { + scrapeReverseDependencies(basicData, postgrestConnector, scrapedAt); + return null; + }; + tasks.add(task); + } + + try (ExecutorService executorService = Executors.newCachedThreadPool()) { + List> completedTasks = executorService.invokeAll(tasks); + for (Future completedTask : completedTasks) { + try { + completedTask.get(); + } catch (ExecutionException | InterruptedException e) { + Utils.saveExceptionInDatabase("Package manager scraper", "package_manager", null, e); + + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + } + } + } catch (InterruptedException e) { + Utils.saveExceptionInDatabase("Package manager scraper", "package_manager", null, e); + Thread.currentThread().interrupt(); + } + + long time = System.currentTimeMillis() - t1; + + LOGGER.info("Done scraping package manager data ({} ms.)", time); + } + + static PackageManagerScraper scraperForType(PackageManagerType type, String url) { + return switch (type) { + case anaconda -> new AnacondaScraper(url); + case cran -> new CranScraper(url); + case crates -> new CratesScraper(url); + case dockerhub -> new DockerHubScraper(url); + case fourtu -> new FourTuScraper(); + case golang -> new GoScraper(url); + case maven, sonatype -> new MavenScraper(url); + case npm -> new NpmScraper(url); + case pypi -> new PypiScraper(url); + case pixi -> new PixiScraper(url); + case other -> throw new RuntimeException("Package manager scraper requested for 'other'"); + }; + } + + static void scrapeDownloads(BasicPackageManagerData data, PostgrestConnector postgrestConnector, ZonedDateTime scrapedAt) { + String packageManagerUrl = data.url(); + PackageManagerType type = data.type(); + + Long downloads; + try { + PackageManagerScraper scraper = scraperForType(type, packageManagerUrl); + downloads = scraper.downloads(); + postgrestConnector.saveDownloadCount(data.id(), downloads, scrapedAt); + } catch (RsdRateLimitException e) { + Utils.saveExceptionInDatabase("Package manager downloads scraper", "package_manager", data.id(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "package_manager", "download_count_last_error", data.id() + .toString(), "id", null, null); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase("Package manager downloads scraper", "package_manager", data.id(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "package_manager", "download_count_last_error", data.id() + .toString(), "id", scrapedAt, "download_count_scraped_at"); + } catch (IOException e) { + Utils.saveExceptionInDatabase("Package manager downloads scraper", "package_manager", data.id(), e); + Utils.saveErrorMessageInDatabase("Unknown error", "package_manager", "download_count_last_error", data.id() + .toString(), "id", scrapedAt, "download_count_scraped_at"); + } + } + + static void scrapeReverseDependencies(BasicPackageManagerData data, PostgrestConnector postgrestConnector, ZonedDateTime scrapedAt) { + String packageManagerUrl = data.url(); + PackageManagerType type = data.type(); + + Integer revDeps; + try { + PackageManagerScraper scraper = scraperForType(type, packageManagerUrl); + revDeps = scraper.reverseDependencies(); + postgrestConnector.saveReverseDependencyCount(data.id(), revDeps, scrapedAt); + } catch (RsdRateLimitException e) { + Utils.saveExceptionInDatabase("Package manager reverse dependencies scraper", "package_manager", data.id(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "package_manager", "reverse_dependency_count_last_error", data.id() + .toString(), "id", null, null); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase("Package manager reverse dependencies scraper", "package_manager", data.id(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), "package_manager", "reverse_dependency_count_last_error", data.id() + .toString(), "id", scrapedAt, "reverse_dependency_count_scraped_at"); + } catch (Exception e) { + Utils.saveExceptionInDatabase("Package manager reverse dependencies scraper", "package_manager", data.id(), e); + Utils.saveErrorMessageInDatabase("Unknown error", "package_manager", "reverse_dependency_count_last_error", data.id() + .toString(), "id", scrapedAt, "reverse_dependency_count_scraped_at"); + } + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PackageManagerType.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PackageManagerType.java new file mode 100644 index 0000000..91fbe03 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PackageManagerType.java @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2023 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2025 Paula Stock (GFZ) +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager; + +public enum PackageManagerType { + anaconda, + cran, + crates, + dockerhub, + fourtu, + golang, + maven, + npm, + pixi, + pypi, + sonatype, + other +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PostgrestConnector.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PostgrestConnector.java new file mode 100644 index 0000000..37273cd --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PostgrestConnector.java @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2025 Paula Stock (GFZ) +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager; + +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Objects; +import java.util.UUID; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import nl.esciencecenter.rsd.scraper.Utils; + +public class PostgrestConnector { + + private final String backendUrl; + + public PostgrestConnector(String backendUrl) { + this.backendUrl = Objects.requireNonNull(backendUrl); + } + + public Collection oldestDownloadCounts(int limit) { + String filter = "download_count_scraping_disabled_reason=is.null&or=(package_manager.eq.dockerhub,package_manager.eq.pixi)"; + String data = Utils.getAsAdmin(backendUrl + "?" + filter + "&select=id,url,package_manager&order=download_count_scraped_at.asc.nullsfirst&limit=" + limit + + "&" + Utils.atLeastOneHourAgoFilter("download_count_scraped_at") + ); + return parseBasicJsonData(data); + } + + public Collection oldestReverseDependencyCounts(int limit) { + String filter = "reverse_dependency_count_scraping_disabled_reason=is.null&or=(package_manager.eq.anaconda,package_manager.eq.cran,package_manager.eq.crates,package_manager.eq.golang,package_manager.eq.maven,package_manager.eq.npm,package_manager.eq.pypi,package_manager.eq.sonatype)"; + String data = Utils.getAsAdmin(backendUrl + "?" + filter + "&select=id,url,package_manager&order=reverse_dependency_count_scraped_at.asc.nullsfirst&limit=" + limit + "&" + Utils.atLeastOneHourAgoFilter("reverse_dependency_count_scraped_at")); + return parseBasicJsonData(data); + } + + public void saveDownloadCount(UUID id, Long count, ZonedDateTime scrapedAt) { + String json = "{\"download_count\": %s, \"download_count_scraped_at\": \"%s\", \"download_count_last_error\": null}".formatted(count, scrapedAt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + Utils.patchAsAdmin(backendUrl + "?id=eq." + id, json); + } + + public void saveReverseDependencyCount(UUID id, Integer count, ZonedDateTime scrapedAt) { + String json = "{\"reverse_dependency_count\": %s, \"reverse_dependency_count_scraped_at\": \"%s\", \"reverse_dependency_count_last_error\": null}".formatted(count, scrapedAt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + Utils.patchAsAdmin(backendUrl + "?id=eq." + id, json); + } + + Collection parseBasicJsonData(String json) { + JsonArray dataInArray = JsonParser.parseString(json).getAsJsonArray(); + Collection result = new ArrayList<>(); + for (JsonElement element : dataInArray) { + JsonObject jsonObject = element.getAsJsonObject(); + String idString = jsonObject.getAsJsonPrimitive("id").getAsString(); + UUID id = UUID.fromString(idString); + String url = jsonObject.getAsJsonPrimitive("url").getAsString(); + PackageManagerType type = PackageManagerType.valueOf(jsonObject.getAsJsonPrimitive("package_manager").getAsString()); + + result.add(new BasicPackageManagerData(id, url, type)); + } + return result; + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/AnacondaScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/AnacondaScraper.java new file mode 100644 index 0000000..581763d --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/AnacondaScraper.java @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; + +import nl.esciencecenter.rsd.scraper.RsdResponseException; + +import java.io.IOException; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class AnacondaScraper implements PackageManagerScraper { + + private final String packageName; + private static final Pattern urlPattern = Pattern.compile("https://anaconda\\.org/[^/]+/([^/]+)/?"); + + public AnacondaScraper(String url) { + Objects.requireNonNull(url); + Matcher urlMatcher = urlPattern.matcher(url); + if (!urlMatcher.matches()) { + throw new RuntimeException("Invalid Anaconda URL: " + url); + } + + packageName = urlMatcher.group(1); + } + + @Override + public Long downloads() { + throw new UnsupportedOperationException(); + } + + @Override + public Integer reverseDependencies() throws IOException, InterruptedException, RsdResponseException { + String data = PackageManagerScraper.doLibrariesIoRequest("https://libraries.io/api/conda/" + packageName); + JsonElement tree = JsonParser.parseString(data); + return tree.getAsJsonObject().getAsJsonPrimitive("dependents_count").getAsInt(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraper.java new file mode 100644 index 0000000..826f4f9 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraper.java @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; + +import nl.esciencecenter.rsd.scraper.RsdResponseException; + +import java.io.IOException; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class CranScraper implements PackageManagerScraper { + + final String packageName; + private static final Pattern urlPattern1 = Pattern.compile("https://cran\\.r-project\\.org/web/packages/([^/\\s]+)/?(index\\.html/?)?", Pattern.CASE_INSENSITIVE); + private static final Pattern urlPattern2 = Pattern.compile("https://cran\\.r-project\\.org/package=([^/\\s]+)/?", Pattern.CASE_INSENSITIVE); + + public CranScraper(String url) { + Objects.requireNonNull(url); + + Matcher urlMatcher1 = urlPattern1.matcher(url); + if (urlMatcher1.matches()) { + packageName = urlMatcher1.group(1); + return; + } + + Matcher urlMatcher2 = urlPattern2.matcher(url); + if (urlMatcher2.matches()) { + packageName = urlMatcher2.group(1); + return; + } + + throw new RuntimeException("Invalid CRAN URL: " + url); + } + + @Override + public Long downloads() { + throw new UnsupportedOperationException(); + } + + @Override + public Integer reverseDependencies() throws IOException, InterruptedException, RsdResponseException { + String data = PackageManagerScraper.doLibrariesIoRequest("https://libraries.io/api/cran/" + packageName); + JsonElement tree = JsonParser.parseString(data); + return tree.getAsJsonObject().getAsJsonPrimitive("dependents_count").getAsInt(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraper.java new file mode 100644 index 0000000..ab3db87 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraper.java @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.RsdResponseException; + +import java.io.IOException; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class CratesScraper implements PackageManagerScraper { + + final String packageName; + private static final Pattern cratesPattern = Pattern.compile("https://crates\\.io/crates/([^/]+)/?"); + + public CratesScraper(String url) { + Objects.requireNonNull(url); + + Matcher cratesMatcher = cratesPattern.matcher(url); + if (cratesMatcher.matches()) { + this.packageName = cratesMatcher.group(1); + return; + } + + throw new RuntimeException("Invalid crates.io URL: " + url); + } + + @Override + public Long downloads() { + throw new UnsupportedOperationException(); + } + + // Example URL: https://libraries.io/api/cargo/tokio + @Override + public Integer reverseDependencies() throws IOException, InterruptedException, RsdResponseException { + String data = PackageManagerScraper.doLibrariesIoRequest("https://libraries.io/api/cargo/" + packageName); + JsonElement tree = JsonParser.parseString(data); + return tree.getAsJsonObject().getAsJsonPrimitive("dependents_count").getAsInt(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/DockerHubScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/DockerHubScraper.java new file mode 100644 index 0000000..f5a034c --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/DockerHubScraper.java @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.RsdResponseException; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class DockerHubScraper implements PackageManagerScraper { + + private final String owner; + private final String packageName; + private static final Pattern urlPattern = Pattern.compile("https://hub\\.docker\\.com/(r/)?([^/]+)/([^/]+)/?"); + + public DockerHubScraper(String url) { + Objects.requireNonNull(url); + Matcher urlMatcher = urlPattern.matcher(url); + if (!urlMatcher.matches()) { + throw new RuntimeException("Invalid Docker Hub URL: " + url); + } + + owner = urlMatcher.group(2); + packageName = urlMatcher.group(3); + } + + @Override + public Long downloads() throws RsdResponseException { + String url; + if (owner.equals("_")) url = "https://hub.docker.com/v2/repositories/library/" + packageName; + else url = "https://hub.docker.com/v2/repositories/" + owner + "/" + packageName; + HttpClient client = HttpClient.newBuilder() + .followRedirects(HttpClient.Redirect.NORMAL) + .build(); + HttpRequest request = HttpRequest.newBuilder(URI.create(url)) + .timeout(Duration.ofSeconds(30)) + .build(); + String json; + try { + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + json = switch (response.statusCode()) { + case 200 -> response.body(); + default -> + throw new RsdResponseException(response.statusCode(), request.uri(), response.body(), "Unexpected response"); + }; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return parseDowloadsResponse(json); + } + + @Override + public Integer reverseDependencies() { + throw new UnsupportedOperationException(); + } + + Long parseDowloadsResponse(String json) { + return JsonParser.parseString(json).getAsJsonObject().getAsJsonPrimitive("pull_count").getAsLong(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/FourTuScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/FourTuScraper.java new file mode 100644 index 0000000..5a42818 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/FourTuScraper.java @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import nl.esciencecenter.rsd.scraper.RsdResponseException; + +import java.io.IOException; + +public class FourTuScraper implements PackageManagerScraper { + @Override + public Long downloads() throws IOException, RsdResponseException { + throw new UnsupportedOperationException(); + } + + @Override + public Integer reverseDependencies() throws IOException, InterruptedException, RsdResponseException { + throw new UnsupportedOperationException(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraper.java new file mode 100644 index 0000000..18f4c1a --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraper.java @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; + +import java.io.IOException; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class GoScraper implements PackageManagerScraper { + + final String packageName; + private static final Pattern goPattern = Pattern.compile("https://pkg\\.go\\.dev/(\\S+?)/?"); + + public GoScraper(String url) { + Objects.requireNonNull(url); + + Matcher goMatcher = goPattern.matcher(url); + if (goMatcher.matches()) { + this.packageName = goMatcher.group(1); + return; + } + + throw new RuntimeException("Invalid Go URL: " + url); + } + + @Override + public Long downloads() { + throw new UnsupportedOperationException(); + } + + // Example URL: https://libraries.io/api/go/github.com%2Fgin-gonic%2Fgin + // Example URL: https://libraries.io/api/go/google.golang.org%2Fgrpc + @Override + public Integer reverseDependencies() throws IOException, InterruptedException, RsdResponseException { + String data = PackageManagerScraper.doLibrariesIoRequest("https://libraries.io/api/go/" + Utils.urlEncode(packageName)); + JsonElement tree = JsonParser.parseString(data); + return tree.getAsJsonObject().getAsJsonPrimitive("dependents_count").getAsInt(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraper.java new file mode 100644 index 0000000..bbd74a5 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraper.java @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; + +import nl.esciencecenter.rsd.scraper.RsdResponseException; + +import java.io.IOException; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class MavenScraper implements PackageManagerScraper { + + final String groupId; + final String artifactId; + private static final Pattern mvnPattern = Pattern.compile("https://mvnrepository\\.com/artifact/([^/]+)/([^/]+)/?"); + private static final Pattern sonatypePattern = Pattern.compile("https://central\\.sonatype\\.com/artifact/([^/]+)/([^/]+)/?"); + + public MavenScraper(String url) { + Objects.requireNonNull(url); + + Matcher mvnMatcher = mvnPattern.matcher(url); + if (mvnMatcher.matches()) { + groupId = mvnMatcher.group(1); + artifactId = mvnMatcher.group(2); + return; + } + + Matcher sonatypeMatcher = sonatypePattern.matcher(url); + if (sonatypeMatcher.matches()) { + groupId = sonatypeMatcher.group(1); + artifactId = sonatypeMatcher.group(2); + return; + } + + throw new RuntimeException("Invalid Maven URL: " + url); + } + + @Override + public Long downloads() { + throw new UnsupportedOperationException(); + } + + // Example URL: https://libraries.io/api/maven/io.github.sanctuuary:APE + @Override + public Integer reverseDependencies() throws IOException, InterruptedException, RsdResponseException { + String data = PackageManagerScraper.doLibrariesIoRequest("https://libraries.io/api/maven/" + groupId + ":" + artifactId); + JsonElement tree = JsonParser.parseString(data); + return tree.getAsJsonObject().getAsJsonPrimitive("dependents_count").getAsInt(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/NpmScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/NpmScraper.java new file mode 100644 index 0000000..de89f2b --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/NpmScraper.java @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; + +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; + +import java.io.IOException; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class NpmScraper implements PackageManagerScraper { + + private final String packageName; + private static final Pattern urlPattern = Pattern.compile("https://www\\.npmjs\\.com/package/(.+?)/?"); + + public NpmScraper(String url) { + Objects.requireNonNull(url); + Matcher urlMatcher = urlPattern.matcher(url); + if (!urlMatcher.matches()) { + throw new RuntimeException("Invalid NPM URL: " + url); + } + + packageName = urlMatcher.group(1); + } + + @Override + public Long downloads() { + throw new UnsupportedOperationException(); + } + + @Override + public Integer reverseDependencies() throws IOException, InterruptedException, RsdResponseException { + String data = PackageManagerScraper.doLibrariesIoRequest("https://libraries.io/api/npm/" + Utils.urlEncode(packageName)); + JsonElement tree = JsonParser.parseString(data); + return tree.getAsJsonObject().getAsJsonPrimitive("dependents_count").getAsInt(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PackageManagerScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PackageManagerScraper.java new file mode 100644 index 0000000..6d8481b --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PackageManagerScraper.java @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdRateLimitException; +import nl.esciencecenter.rsd.scraper.RsdResponseException; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.Optional; + +public interface PackageManagerScraper { + + Long downloads() throws IOException, RsdResponseException; + + Integer reverseDependencies() throws IOException, InterruptedException, RsdResponseException; + + static String doLibrariesIoRequest(String url) throws IOException, InterruptedException, RsdResponseException { + Optional optionalApiKey = Config.librariesIoKey(); + if (optionalApiKey.isPresent()) url += "?api_key=" + optionalApiKey.get(); + + + HttpRequest request = HttpRequest.newBuilder(URI.create(url)) + .timeout(Duration.ofSeconds(30)) + .build(); + try (HttpClient client = HttpClient.newBuilder() + .followRedirects(HttpClient.Redirect.NORMAL) + .build();) { + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + return switch (response.statusCode()) { + case 429 -> + throw new RsdRateLimitException(429, request.uri(), response.body(), "Rate limit reached for libraries.io"); + case 404 -> + throw new RsdResponseException(404, request.uri(), response.body(), "Not found, is the URL correct?"); + case 200 -> response.body(); + default -> + throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response"); + }; + } + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PixiScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PixiScraper.java new file mode 100644 index 0000000..6c8ff1d --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PixiScraper.java @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2025 Paula Stock (GFZ) +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; + +public class PixiScraper implements PackageManagerScraper { + + private final String channelName; + private final String packageName; + private static final Pattern urlPattern = Pattern.compile("https://prefix\\.dev/channels/([^/]+)/packages/([^/]+)"); + + public PixiScraper(String url) { + Objects.requireNonNull(url); + Matcher urlMatcher = urlPattern.matcher(url); + if (!urlMatcher.matches()) { + throw new RuntimeException("Invalid Pixi URL: " + url); + } + + channelName = urlMatcher.group(1); + packageName = urlMatcher.group(2); + } + + @Override + public Long downloads() throws RsdResponseException { + String graphqlQuery = String.format(""" + { + package(channelName: "%s", name: "%s") { + downloadCounts { + version + count + } + } + } + """, + channelName, + packageName); + JsonObject body = new JsonObject(); + body.addProperty("query", graphqlQuery); + String response = Utils.post("https://prefix.dev/api/graphql", body.toString(), "Content-Type", "application/json"); + JsonObject jsonObject = JsonParser.parseString(response).getAsJsonObject(); + JsonArray countItems = jsonObject.getAsJsonObject("data").getAsJsonObject("package").getAsJsonArray("downloadCounts"); + + long downloadsSum = 0; + for (JsonElement item : countItems) { + int count = item.getAsJsonObject().get("count").getAsInt(); + downloadsSum += count; + } + + return downloadsSum; + } + + @Override + public Integer reverseDependencies() { + throw new UnsupportedOperationException(); + } + +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PypiScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PypiScraper.java new file mode 100644 index 0000000..9bd0765 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PypiScraper.java @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; + +import nl.esciencecenter.rsd.scraper.RsdResponseException; + +import java.io.IOException; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class PypiScraper implements PackageManagerScraper { + + private final String packageName; + private static final Pattern urlPattern = Pattern.compile("https://pypi\\.org/project/([^/]+)/?"); + + public PypiScraper(String url) { + Objects.requireNonNull(url); + Matcher urlMatcher = urlPattern.matcher(url); + if (!urlMatcher.matches()) { + throw new RuntimeException("Invalid PyPi URL: " + url); + } + + packageName = urlMatcher.group(1); + } + + @Override + public Long downloads() { + throw new UnsupportedOperationException(); + } + + @Override + public Integer reverseDependencies() throws IOException, InterruptedException, RsdResponseException { + String data = PackageManagerScraper.doLibrariesIoRequest("https://libraries.io/api/pypi/" + packageName); + JsonElement tree = JsonParser.parseString(data); + return tree.getAsJsonObject().getAsJsonPrimitive("dependents_count").getAsInt(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/MainRor.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/MainRor.java new file mode 100644 index 0000000..7a6d528 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/MainRor.java @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.ror; + +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.concurrent.CompletableFuture; + +public class MainRor { + + private static final Logger LOGGER = LoggerFactory.getLogger(MainRor.class); + private static final int SCRAPING_LIMIT = Config.maxRequestsRor(); + + public static void main(String[] args) { + LOGGER.info("Start scraping ROR data."); + long t1 = System.currentTimeMillis(); + scrapeLocationData(); + long time = System.currentTimeMillis() - t1; + LOGGER.info("Done scraping ROR data ({} ms).", time); + } + + private static void scrapeLocationData() { + RorPostgrestConnector organisationsInRSD = new RorPostgrestConnector(); + Collection organisationsToScrape = organisationsInRSD.organisationsWithoutLocation(SCRAPING_LIMIT); + CompletableFuture[] futures = new CompletableFuture[organisationsToScrape.size()]; + ZonedDateTime scrapedAt = ZonedDateTime.now(); + int i = 0; + String tableName = "organisation"; + String columnName = "ror_last_error"; + String primaryKeyName = "id"; + String scrapedAtName = "ror_scraped_at"; + for (OrganisationDatabaseData organisation : organisationsToScrape) { + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + RorScraper rorScraper = new RorScraper(organisation.rorId()); + RorData data = rorScraper.scrapeData(); + OrganisationDatabaseData updatedOrganisationDatabaseData = new OrganisationDatabaseData(organisation.id(), organisation.rorId(), scrapedAt, data); + organisationsInRSD.saveLocationData(updatedOrganisationDatabaseData); + } catch (RsdResponseException | IOException | InterruptedException e) { + Utils.saveExceptionInDatabase("ROR location scraper", tableName, organisation.id(), e); + Utils.saveErrorMessageInDatabase(e.getMessage(), tableName, columnName, organisation.id().toString(), primaryKeyName, scrapedAt, scrapedAtName); + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + } catch (Exception e) { + Utils.saveExceptionInDatabase("ROR location scraper", tableName, organisation.id(), e); + Utils.saveErrorMessageInDatabase("Unknown error", tableName, columnName, organisation.id().toString(), primaryKeyName, scrapedAt, scrapedAtName); + } + }); + futures[i] = future; + i++; + } + CompletableFuture.allOf(futures).join(); + } + +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/OrganisationDatabaseData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/OrganisationDatabaseData.java new file mode 100644 index 0000000..9684f77 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/OrganisationDatabaseData.java @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.ror; + +import java.time.ZonedDateTime; +import java.util.UUID; + +public record OrganisationDatabaseData( + UUID id, + RorId rorId, + ZonedDateTime rorScrapedAt, + RorData data +) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorData.java new file mode 100644 index 0000000..4845f86 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorData.java @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.ror; + +import java.util.List; + +public record RorData( + String country, + String city, + String wikipediaUrl, + List rorTypes, + List rorNames, + Double lat, + Double lon +) { +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorId.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorId.java new file mode 100644 index 0000000..4a098ce --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorId.java @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.ror; + +import java.net.URI; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * This class represents ROR identifiers as described in the ROR documentation. + *

+ * A {@link #isValidRorUrl(String) static factory method} is provided to construct instances of this class from a full URL string. + * As such, any instance of this class is guaranteed to contain a syntactically valid ROR ID. + * However, the last two digits of a ROR ID should be a checksum; we don't currently validate this checksum. + */ +public class RorId { + + private static final String ROR_BASE_URL = "https://ror.org/"; + // https://ror.org/blog/2024-04-15-announcing-ror-v2/ + private static final String ROR_BASE_API_V1_URL = "https://api.ror.org/v1/organizations/"; + private static final Pattern ROR_URL_PATTERN = Pattern.compile("^https://ror\\.org/(0[a-hj-km-np-tv-z|\\d]{6}\\d{2})$"); + + private final String id; + + private RorId(String id) { + this.id = id; + } + + /** + * Tests if a URL string is a valid ROR ID as described in the ROR documentation. + * The checksum is not validated though. + * + * @param url the URL string to validate + * @return whether or not the provided URL is a valid ROR ID + */ + public static boolean isValidRorUrl(String url) { + return url != null && ROR_URL_PATTERN.asPredicate().test(url); + } + + /** + * A factory method to create instanced of RorId. + * Only full URLs that look like {@code https://ror.org/02mhbdp94} are accepted. + *

+ * Callers of this method should call {@link #isValidRorUrl(String)} first to validate their URL and + * thus to prevent an exception from being thrown. + * + * @param url a non-null URL string that satisfies the conditions in the ROR documentation + * @return a non-null instance of a RorId + * @throws NullPointerException when {@code url} is {@code null} + * @throws IllegalArgumentException when {@code url} is not a valid ROR ID + */ + public static RorId fromUrlString(String url) { + Objects.requireNonNull(url); + if (!isValidRorUrl(url)) { + throw new IllegalArgumentException("The url %s is not a valid ROR ID".formatted(url)); + } + + Matcher matcher = ROR_URL_PATTERN.matcher(url); + matcher.find(); + String id = matcher.group(1); + return new RorId(id); + } + + public URI asUrl() { + return URI.create(ROR_BASE_URL + id); + } + + public URI asApiV1Url() { + return URI.create(ROR_BASE_API_V1_URL + id); + } + + @Override + public String toString() { + return ROR_BASE_URL + id; + } + + @Override + public int hashCode() { + return id.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (this == other) { + return true; + } + if (other instanceof RorId otherRorId) { + return id.equals(otherRorId.id); + } + + return false; + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnector.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnector.java new file mode 100644 index 0000000..6a992e4 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnector.java @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: 2023 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.ror; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.Utils; + +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.UUID; + +public class RorPostgrestConnector { + private final String backendUrl; + + public RorPostgrestConnector() { + this.backendUrl = Config.backendBaseUrl(); + } + + public Collection organisationsWithoutLocation(int limit) { + String filter = "organisation?ror_id=not.is.null&order=ror_scraped_at.asc.nullsfirst&select=id,ror_id&limit=" + limit; + String data = Utils.getAsAdmin(backendUrl + "/" + filter); + return parseBasicJsonData(data); + } + + static Collection parseBasicJsonData(String data) { + JsonArray dataInArray = JsonParser.parseString(data).getAsJsonArray(); + Collection result = new ArrayList<>(); + for (JsonElement element : dataInArray) { + JsonObject organisationJson = element.getAsJsonObject(); + UUID id = UUID.fromString(organisationJson.getAsJsonPrimitive("id").getAsString()); + RorId rorId = RorId.fromUrlString(organisationJson.getAsJsonPrimitive("ror_id").getAsString()); + result.add(new OrganisationDatabaseData(id, rorId, null, null)); + } + return result; + } + + public void saveLocationData(OrganisationDatabaseData organisationData) { + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("country", organisationData.data().country()); + jsonObject.addProperty("city", organisationData.data().city()); + jsonObject.addProperty("wikipedia_url", organisationData.data().wikipediaUrl()); + + JsonArray rorTypesJsonArray = new JsonArray(); + List rorTypes = organisationData.data().rorTypes(); + if (rorTypes != null) { + for (String rorType : rorTypes) { + rorTypesJsonArray.add(rorType); + } + } + jsonObject.add("ror_types", rorTypesJsonArray); + + JsonArray rorNamesJsonArray = new JsonArray(); + List rorNames = organisationData.data().rorNames(); + if (rorNames != null) { + for (String rorName : rorNames) { + rorNamesJsonArray.add(rorName); + } + } + jsonObject.add("ror_names", rorNamesJsonArray); + + jsonObject.addProperty("lat", organisationData.data().lat()); + jsonObject.addProperty("lon", organisationData.data().lon()); + + jsonObject.addProperty("ror_scraped_at", organisationData.rorScrapedAt() + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + jsonObject.add("ror_last_error", JsonNull.INSTANCE); + + Utils.patchAsAdmin(backendUrl + "/organisation?id=eq." + organisationData.id(), jsonObject.toString()); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorScraper.java new file mode 100644 index 0000000..61c2e37 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorScraper.java @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: 2023 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.ror; + +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public class RorScraper { + + private final RorId rorId; + + public RorScraper(RorId rorId) { + this.rorId = Objects.requireNonNull(rorId); + } + + private String getFromApi() throws IOException, InterruptedException, RsdResponseException { + // e.g https://api.ror.org/organizations/04tsk2644 + return Utils.get(rorId.asApiV1Url().toString()); + } + + public RorData scrapeData() throws RsdResponseException, IOException, InterruptedException { + String json = getFromApi(); + return parseData(json); + } + + static RorData parseData(String json) { + JsonElement jsonElement = JsonParser.parseString(json); + final String addressesKey = "addresses"; + + String country = Utils.safelyGetOrNull(jsonElement, j -> j + .getAsJsonObject() + .getAsJsonObject("country") + .getAsJsonPrimitive("country_name") + .getAsString()); + String city = Utils.safelyGetOrNull(jsonElement, j -> j + .getAsJsonObject() + .getAsJsonArray(addressesKey) + .get(0) + .getAsJsonObject() + .getAsJsonPrimitive("city") + .getAsString()); + String wikipediaUrl = Utils.safelyGetOrNull(jsonElement, j -> { + JsonElement wikiElement = j + .getAsJsonObject() + .get("wikipedia_url"); + return wikiElement.isJsonPrimitive() ? wikiElement.getAsString() : null; + }); + if (wikipediaUrl != null && wikipediaUrl.isBlank()) { + wikipediaUrl = null; + } + List rorTypes = Utils.safelyGetOrNull(jsonElement, j -> j + .getAsJsonObject() + .getAsJsonArray("types") + .asList() + .stream() + .map(JsonElement::getAsString) + .toList() + ); + + List rorNames = new ArrayList<>(); + List rorAliases = Utils.safelyGetOrNull(jsonElement, j -> j + .getAsJsonObject() + .getAsJsonArray("aliases") + .asList() + .stream() + .map(alias -> alias.getAsJsonPrimitive().getAsString()) + .toList() + ); + if (rorAliases != null) { + rorNames.addAll(rorAliases); + } + List rorAcronyms = Utils.safelyGetOrNull(jsonElement, j -> j + .getAsJsonObject() + .getAsJsonArray("acronyms") + .asList() + .stream() + .map(acronym -> acronym.getAsJsonPrimitive().getAsString()) + .toList() + ); + if (rorAcronyms != null) { + rorNames.addAll(rorAcronyms); + } + List rorLabels = Utils.safelyGetOrNull(jsonElement, j -> j + .getAsJsonObject() + .getAsJsonArray("labels") + .asList() + .stream() + .map(label -> label.getAsJsonObject().getAsJsonPrimitive("label").getAsString()) + .toList() + ); + if (rorLabels != null) { + rorNames.addAll(rorLabels); + } + + Double lat = Utils.safelyGetOrNull(jsonElement, j -> j + .getAsJsonObject() + .getAsJsonArray(addressesKey) + .get(0) + .getAsJsonObject() + .getAsJsonPrimitive("lat") + .getAsDouble()); + Double lon = Utils.safelyGetOrNull(jsonElement, j -> j + .getAsJsonObject() + .getAsJsonArray(addressesKey) + .get(0) + .getAsJsonObject() + .getAsJsonPrimitive("lng") + .getAsDouble()); + + return new RorData( + country, + city, + wikipediaUrl, + rorTypes == null ? Collections.emptyList() : rorTypes, + rorNames, + lat, + lon + ); + } +} diff --git a/scrapers/src/main/resources/logback.xml b/scrapers/src/main/resources/logback.xml new file mode 100644 index 0000000..a94f683 --- /dev/null +++ b/scrapers/src/main/resources/logback.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} -%kvp- %msg%n + + + + + + + diff --git a/scrapers/src/main/resources/oaipmh.py b/scrapers/src/main/resources/oaipmh.py new file mode 100644 index 0000000..6179bae --- /dev/null +++ b/scrapers/src/main/resources/oaipmh.py @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2022 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +import json +import xml.etree.ElementTree as Tree +import os +from datetime import datetime, timedelta +import requests +from requests.exceptions import HTTPError +import time +import jwt + + +# Still need to enrich and correct the metadata. Currently serving Zenodo's metadata as-is. +def list_records(dois): + oaipmh_elem = _build_oaipmh_elem() + + # get the conceptdois from /api/software + backend_url = os.environ.get('POSTGREST_URL') + response = requests.get(backend_url + '/software?select=concept_doi') + + if dois is None: + softwares = response.json() + else: + softwares = [software for software in response.json() if software["concept_doi"] in dois] + + headers = { + 'Authorization': 'Bearer ' + os.environ.get('ZENODO_ACCESS_TOKEN') + } + + n_softwares = len(softwares) + + for i_software, software in enumerate(softwares): + + if software["concept_doi"] is None: + print(" %d/%d: concept_doi is None" % (i_software + 1, n_softwares)) + continue + + try: + redirect_url = _get_redirect(software) + identifier = _get_zenodo_identifier(redirect_url, headers) + url = 'https://zenodo.org/oai2d?verb=GetRecord&identifier=oai:zenodo.org:' + identifier + \ + '&metadataPrefix=datacite4' + record_elem = _get_datacite(url, headers) + oaipmh_elem.find('{http://www.openarchives.org/OAI/2.0/}ListRecords') \ + .append(record_elem) + + print( + " %d/%d: retrieved datacite4 metadata for %s" % (i_software + 1, n_softwares, software["concept_doi"]) + ) + + except requests.exceptions.RequestException as e: + print( + " %d/%d: There was an error while retrieving OAI-PMH metadata for https://doi.org/%s. %s" % + (i_software + 1, n_softwares, software["concept_doi"], str(e)) + ) + continue + + document = Tree.ElementTree(oaipmh_elem) + xml_str = Tree.tostring(document.getroot(), encoding='UTF-8', method='xml', xml_declaration=True).decode("utf8") + data_dict = {"data": xml_str} + + jwt_secret = os.environ.get('PGRST_JWT_SECRET') + jwt_token = jwt.encode({"role": "rsd_admin", "exp": datetime.now() + timedelta(minutes = 1)}, jwt_secret, algorithm="HS256") + patch_response = requests.patch('{}/oaipmh?id=is.true'.format(backend_url), json.dumps(data_dict), headers={'Authorization': 'Bearer ' + jwt_token}) + if patch_response.status_code != 204: + print(patch_response.text) + patch_response.raise_for_status() + + +def _build_oaipmh_elem(): + root_elem = Tree.Element('{http://www.openarchives.org/OAI/2.0/}OAI-PMH') + + response_date_elem = Tree.Element('{http://www.openarchives.org/OAI/2.0/}responseDate') + response_date_elem.text = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' + root_elem.append(response_date_elem) + + request_elem = Tree.Element('{http://www.openarchives.org/OAI/2.0/}request') + request_elem.set('verb', 'ListRecords') + request_elem.set('metadataPrefix', 'datacite4') + request_elem.text = 'https://zenodo.org/oai2d' + root_elem.append(request_elem) + + listrecords_elem = Tree.Element('{http://www.openarchives.org/OAI/2.0/}ListRecords') + root_elem.append(listrecords_elem) + return root_elem + + +def _get_redirect(software): + response = requests.head('https://doi.org/{conceptdoi}'.format(conceptdoi=software["concept_doi"])) + if response.status_code == 302: + return response.next.url + elif response.status_code == 429: + print("HttpError 429 while trying to redirect from doi.org.") + response.raise_for_status() + else: + raise HTTPError( + "Expected a redirect from doi.org to zenodo.org, got {0} instead." + .format(response.status_code) + ) + + +def _get_zenodo_identifier(redirect_url, headers): + response = requests.head(redirect_url, headers=headers) + + if response.status_code == 302: + return response.next.url.split('/')[-1:][0] + elif response.status_code == 429: + print("HttpError 429 while trying to retrieve a record from zenodo.org.") + epoch_rate_limit_reset = int(response.headers.get("X-RateLimit-Reset")) + epoch_now = time.time() + sleep_time = max(0, epoch_rate_limit_reset - epoch_now) + print("Sleeping until: {} ({} seconds)".format(epoch_rate_limit_reset, sleep_time)) + time.sleep(sleep_time) + return _get_zenodo_identifier(redirect_url, headers) + else: + raise HTTPError( + "Expected a redirect from a conceptdoi to a versioned doi, got {0} instead." + .format(response.status_code) + ) + + +def _get_datacite(url, headers): + response = requests.get(url, headers=headers) + + if response.status_code != requests.codes.ok: + print("Error while trying to retrieve datacite metadata from zenodo.org.") + response.raise_for_status() + + # future work: correct/expand the datacite4 representation with other data from the RSDjson + return Tree.fromstring(response.text) \ + .find("{http://www.openarchives.org/OAI/2.0/}GetRecord") \ + .find("{http://www.openarchives.org/OAI/2.0/}record") + + +if __name__ == "__main__": + if bool(os.environ.get('ENABLE_OAIPMH_SCRAPER')): + list_records(None) + else: + print("The oaipmh scraper is disabled, aborting") diff --git a/scrapers/src/main/resources/releases.py b/scrapers/src/main/resources/releases.py new file mode 100644 index 0000000..0169199 --- /dev/null +++ b/scrapers/src/main/resources/releases.py @@ -0,0 +1,310 @@ +# SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2022 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +from datetime import datetime +import re +import os +import time + +import requests +from cffconvert import Citation +import jwt +from datetime import datetime, timedelta +import json + + +class ReleaseScraper: + + """ + 1. check that the doi is a zenodo doi + 2. check that the doi is a concept doi + 3. retrieve versioned dois + foreach versioned doi + 4. visit the github release + 5. on github, check if there is a cff + foreach cff + 6. validate the cff + 7. generate bibtex, ris, endnote, and schema.org strings + + After initialization, the ReleaseScraper instance has a field .releases which is + an array of objects with one of the following layouts: + { + "citability": "full" + "datePublished": "2018-03-16" + "doi": "10.5281/zenodo.1200251", + "files": { + "bibtex": "file contents", + "cff": "file contents", + "schema_dot_org": "file contents", + "endnote": "file contents", + "ris": "file contents" + }, + "tag": "2.6.1" + } + or in case there is no valid CITATION.cff for a given release: + { + "citability": "doi-only" + "datePublished": "2018-02-23" + "doi": "10.5281/zenodo.1183426", + "files": {}, + "tag": "2.4.0" + } + """ + + def __init__(self, doi): + self.latest_schema_dot_org = None + self.releases = list() + self.is_citable = False + self.zenodo_data = dict(conceptdoi=None, versioned_dois=None) + self.title = None + self.message = None + self.doi = None + + if doi is None: + self.message = "record has no doi value of any kind." + return + + if not isinstance(doi, str): + self.message = "doi should be a string." + return + + if doi == "": + self.message = "doi should not be an empty string." + return + + url = "https://doi.org/{0}".format(doi) + response = requests.get(url) + if response.status_code == requests.codes.ok: + self.is_citable = True + self.doi = doi + else: + self.message = "error {0} resolving doi.".format(response.status_code) + return + + if not self.is_zenodo_doi(): + self.message = "doi is not a Zenodo doi." + return + + self.fetch_zenodo_data_conceptdoi() + if not self.is_concept_doi(): + self.message = "doi is not a concept doi." + return + + self.fetch_zenodo_data_versioned_dois() + if "hits" not in self.zenodo_data["versioned_dois"]: + self.message = "record doesn't seem to be resulting from GitHub-Zenodo integration." + return + + self.filter_zenodo_data_versioned_dois() + self.reverse_sort_zenodo_data_versioned_dois() + self.generate_files() + if True not in [release["citability"] == "full" for release in self.releases]: + self.message = "no valid CITATION.cff found in any release." + return + self.determine_latest_schema_dot_org() + self.message = "OK" + + def determine_latest_schema_dot_org(self): + for release in self.releases: + if "schema_dot_org" in release["files"].keys(): + self.latest_schema_dot_org = release["files"]["schema_dot_org"] + return self + + def filter_zenodo_data_versioned_dois(self): + def select_github_url(): + github_link = None + for identifier in hit["metadata"]["related_identifiers"]: + if identifier["scheme"] == "url" and \ + identifier["relation"] == "isSupplementTo" and \ + identifier["identifier"].startswith("https://github.com/"): + github_link = identifier["identifier"] + else: + # Associated Zenodo record does not contain a related identifier with "isSupplementTo" relation + # that points to GitHub. Continue with the next related identifier: + continue + return github_link + + hits = self.zenodo_data["versioned_dois"]["hits"]["hits"] + # already include the "files" key here, even though we're only filling it later: + files = dict() + for hit in hits: + try: + doi = hit["metadata"]["doi"] + date_published = hit["metadata"]["publication_date"] + url = select_github_url() + if url is not None: + tag = re.sub("^.*(/tree/)", "", url) + self.releases.append({ + "citability": "doi-only", + "datePublished": date_published, + "doi": doi, + "files": files, + "tag": tag, + "url": url}) + except Exception as e: + print("Something unexpected happened when trying to retrieve versioned doi info" + + " but I'll try to continue. {0}".format(e)) + return self + + def fetch_zenodo_data_conceptdoi(self): + zenodo_id = self.doi.replace("10.5281/zenodo.", "") + url = "https://zenodo.org/api/records/" + zenodo_id + headers = { + 'Authorization': 'Bearer ' + os.environ.get('ZENODO_ACCESS_TOKEN') + } + r = requests.get(url, headers=headers) + r.raise_for_status() + self.zenodo_data["conceptdoi"] = r.json() + return self + + def fetch_zenodo_data_versioned_dois(self): + url = "https://zenodo.org/api/records?q=conceptdoi:\"{0}\"&all_versions=true&size=100".format(self.doi) + headers = { + 'Authorization': 'Bearer ' + os.environ.get('ZENODO_ACCESS_TOKEN') + } + r = requests.get(url, headers=headers) + r.raise_for_status() + self.zenodo_data["versioned_dois"] = r.json() + hits = self.zenodo_data["versioned_dois"]["hits"]["hits"] + hits_sorted = sorted(hits, key=lambda hit: hit["metadata"]["publication_date"]) + self.title = hits_sorted[-1]["metadata"]["title"] + return self + + def generate_files(self): + for release in self.releases: + try: + override = { + "doi": release["doi"], + "date-released": datetime.strptime(release["datePublished"], "%Y-%m-%d").date(), + "version": release["tag"] + } + remove = ["commit"] + citation = Citation(url=release["url"], override=override, remove=remove) + try: + release["files"] = dict({ + "bibtex": citation.as_bibtex(), + "codemeta": citation.as_codemeta(), + "cff": citation.cffstr, + "schema_dot_org": citation.as_schema_dot_org(), + "endnote": citation.as_enw(), + "ris": citation.as_ris() + }) + release["citability"] = "full" + except Exception: + continue + except Exception as e: + if str(e).startswith("Error requesting file: "): + continue + elif str(e).startswith("Provided CITATION.cff does not seem valid YAML."): + continue + elif type(e).__name__ == "ScannerError": + continue + else: + print("Something went wrong while generating a file for a release:") + print(e) + print("Continuing") + continue + return self + + def is_concept_doi(self): + return "conceptdoi" in self.zenodo_data["conceptdoi"] and \ + self.doi == self.zenodo_data["conceptdoi"]["conceptdoi"] + + def is_zenodo_doi(self): + return self.doi.startswith("10.5281/zenodo.") + + def reverse_sort_zenodo_data_versioned_dois(self): + self.releases.sort(key=lambda release: release["tag"], reverse=True) + return self + + +def get_citations(dois_data): + n_dois = len(dois_data) + + for i_dois, data in enumerate(dois_data): + try: + release_id = data["release_id"] + #first create the entry in the mention table if it doesn't exist yet: + if release_id is None: + backend_release_response = requests.post('{}/release'.format(backend_url), '{{"software": "{}"}}'.format(data["software_id"]), headers={'Authorization': 'Bearer ' + jwt_token, 'Prefer': 'return=representation'}) + release_id = backend_release_response.json()[0]["id"] + + #now set the releases_scraped_at so the release gets pushed back in the scraping order + requests.patch('{}/release?id=eq.{}'.format(backend_url,release_id), '{{"releases_scraped_at": "{}"}}'.format(datetime.now()), headers={'Authorization': 'Bearer ' + jwt_token}) + + doi = data["concept_doi"] + release = ReleaseScraper(doi) + if release.is_citable: + document = { + "is_citable": release.is_citable, + "latest_schema_dot_org": release.latest_schema_dot_org + } + backend_release_response = requests.patch('{}/release?id=eq.{}'.format(backend_url,release_id), document, headers={'Authorization': 'Bearer ' + jwt_token}) + try: + backend_release_response.raise_for_status() + except requests.HTTPError as e: + print(e.response.text) + raise e + if release_id is None: + release_id = backend_release_response.json()[0]["id"] + + release_data = release.releases + release_data_to_save = list() + for release_entry in release_data: + release_content = { + "release_id": release_id, + "citability": release_entry["citability"], + "date_published": release_entry["datePublished"], + "doi": release_entry["doi"], + "tag": release_entry["tag"], + "url": release_entry["url"] + } + if release_entry["citability"] == "full": + release_content["bibtex"] = release_entry["files"]["bibtex"], + release_content["cff"] = release_entry["files"]["cff"], + release_content["codemeta"] = release_entry["files"]["codemeta"], + release_content["endnote"] = release_entry["files"]["endnote"], + release_content["ris"] = release_entry["files"]["ris"], + release_content["schema_dot_org"] = release_entry["files"]["schema_dot_org"] + else: + release_content["bibtex"] = None, + release_content["cff"] = None, + release_content["codemeta"] = None, + release_content["endnote"] = None, + release_content["ris"] = None, + release_content["schema_dot_org"] = None + + release_data_to_save.append(release_content) + + backend_release_content_response = requests.post('{}/release_content?on_conflict=doi'.format(backend_url), json=release_data_to_save, headers={'Authorization': 'Bearer ' + jwt_token, 'Prefer': 'resolution=merge-duplicates'}) + try: + backend_release_content_response.raise_for_status() + except requests.HTTPError as e: + print(e.response.text) + raise e + + if release.message == "OK": + print("{0}/{1} \"{2}\" ({3}): {4}".format(i_dois + 1, n_dois, doi, + release.title, release.message)) + else: + print('{0}/{1}: {2} {3}'.format(i_dois + 1, n_dois, doi, + release.message)) + except BaseException as e: + print("Something went wrong with getting the releases for software with slug " + data["slug"] + ":") + print(e) + print("Continuing with the next release") + + +if __name__ == "__main__": + print("Start scraping releases") + backend_url = os.environ.get('POSTGREST_URL') + number_releases_to_scrape = os.environ.get('MAX_REQUESTS_GITHUB', default='6') + jwt_secret = os.environ.get('PGRST_JWT_SECRET') + jwt_token = jwt.encode({"role": "rsd_admin", "exp": datetime.now() + timedelta(minutes = 10)}, jwt_secret, algorithm="HS256") + response_dois = requests.get('{}/rpc/software_join_release?concept_doi=not.is.null&order=releases_scraped_at.nullsfirst&limit={}'.format(backend_url, number_releases_to_scrape), headers={'Authorization': 'Bearer ' + jwt_token}) + dois_data = response_dois.json() + github_api_token = os.environ.get('API_CREDENTIALS_GITHUB') + get_citations(dois_data) + print("Done scraping releases") diff --git a/scrapers/src/main/resources/requirements.txt b/scrapers/src/main/resources/requirements.txt new file mode 100644 index 0000000..90f97bb --- /dev/null +++ b/scrapers/src/main/resources/requirements.txt @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2022 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +beautifulsoup4==4.6.0 +requests>=2.21.0 +python-dateutil==2.6.1 +Pyzotero==1.5.2 +cffconvert==1.3.3 +PyJWT==2.3.0 diff --git a/scrapers/src/main/resources/zotero.py b/scrapers/src/main/resources/zotero.py new file mode 100644 index 0000000..a331ee8 --- /dev/null +++ b/scrapers/src/main/resources/zotero.py @@ -0,0 +1,187 @@ +# SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2022 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +""" +https://pyzotero.readthedocs.io/en/latest/#retrieving-version-information + +""" + +import logging +from dateutil.parser import parse +import requests +from bs4 import BeautifulSoup +from pyzotero import zotero +import os +import jwt +from datetime import datetime, timedelta + + +def get_project_keys(client): + collections = client.everything(client.collections_top()) + projects_collection = [c for c in collections if c['data']['name'] == 'Projects'][0] + projects = client.everything(client.collections_sub(projects_collection['key'])) + return list(map(lambda x: x['key'], projects)) + + +def get_last_version(): + last_version_item = requests.get('{}/mention?select=version&order=version.desc&limit=1'.format(backend_url), headers={'Authorization': 'Bearer ' + jwt_token}).json() + return 0 if len(last_version_item) == 0 else last_version_item[0]['version'] + + +# DOIs can be either an URL or just the identifier, so make sure its a URL +def doi_to_url(doi): + if doi[:4] == 'http': + return doi + else: + return 'https://doi.org/' + doi + + +# URL is constructed from doi (either as doi field or in 'extra'), or else from the 'url' field +def get_url_for_zotero_item(item): + if 'DOI' in item['data'] and item['data']['DOI']: + return doi_to_url(item['data']['DOI']) + else: + for extra in item['data']['extra'].split('\n'): + if extra[:3] == 'doi' or extra[:3] == 'DOI': + return doi_to_url(extra[5:]) + if 'url' in item['data'] and item['data']['url']: + return item['data']['url'] + return None + + +def get_blog_fields(zotero_item): + try: + data = requests.get(zotero_item['data']['url']).text + soup = BeautifulSoup(data, 'html.parser') + authors = soup.find_all('meta', attrs={'name': 'author'}) + author = authors[0].attrs["content"] if len(authors) > 0 else None + image = soup.find("meta", property="og:image").attrs["content"] + return author, image + except: + return None, None + + +def get_mentions(since_version=None, keys_data=None): + by_version = since_version is not None + by_key = keys_data is not None + + if by_version and by_key: + raise Exception("Use either 'since_version' or 'keys_data', not both") + + client = zotero.Zotero(zotero_library, 'group', zotero_api_token) + + if by_key: + keys = ",".join([k["zotero_key"] for k in keys_data]) + items = client.everything(client.items(itemKey=keys)) + print('Found %d items in Zotero library %s based on supplied key(s).' % (len(items), + zotero_library)) + else: + their_last_version = client.last_modified_version() + our_last_version = get_last_version() + print(('Database collection \'mention\' is currently at version %d; Zotero library %s is cur' + + 'rently at version %d.') % (our_last_version, zotero_library, their_last_version)) + if since_version is None: + since_version = our_last_version + items = (client.everything(client.items(since=since_version))) + + print('Found %d new or updated items in Zotero library %s since version %d.' % (len(items), + zotero_library, + since_version)) + + items_to_save = [] + + project_keys = get_project_keys(client) + supported_types = ['attachment', 'blogPost', 'book', 'bookSection', 'computerProgram', 'conferencePaper', + 'document', 'interview', 'journalArticle', 'magazineArticle', 'manuscript', 'newspaperArticle', + 'note', 'presentation', 'radioBroadcast', 'report', 'thesis', 'videoRecording', 'webpage'] + + n_items = len(items) + for item_index, item in enumerate(items): + + if 'title' not in item['data'] or not item['data']['title']: + print("{0}/{1}: {2} does not have a title." + .format(item_index+1, n_items, item['key'])) + continue + item_collection_keys = item['data'].get('collections', []) + if len(set.intersection(set(item_collection_keys), set(project_keys))) == 0: + print("{0}/{1}: {2} is not part of a project ({3})." + .format(item_index + 1, n_items, item['key'], item["data"]["title"])) + continue + + try: + item_date = parse(item['data']['date']).isoformat()[:19] + 'Z' + except: + print("{0}/{1}: {2} has a date problem ({3})." + .format(item_index + 1, n_items, item['key'], item["data"]["title"])) + continue + + if item["data"]["itemType"] not in supported_types: + print("{0}/{1}: {2} not a supported type ({3})." + .format(item_index + 1, n_items, item['key'], item["data"]["title"])) + continue + + # item is good as far as we know + to_save = { + 'version': item['version'], + 'title': item['data'].get('title', ''), + 'type': item['data']['itemType'], + 'zotero_key': item['key'], + 'is_featured': False, + 'date': item_date, + 'scraped_at': str(datetime.now()), + 'author': None, + 'image': None, + 'url': None + } + url = get_url_for_zotero_item(item) + if url: + to_save['url'] = url + + if item['data']['url'] and '://blog.esciencecenter.nl/' in item['data']['url']: + (author, image) = get_blog_fields(item) + if author is None: + print("{0}/{1}: {2} cannot scrape the author from blog.esciencecenter.nl ({3})." + .format(item_index + 1, n_items, item['key'], item['data']['url'])) + continue + if image is None: + print("{0}/{1}: {2} cannot scrape the image from blog.esciencecenter.nl ({3})." + .format(item_index + 1, n_items, item['key'], item['data']['url'])) + continue + + to_save['is_featured'] = True + to_save['author'] = author + to_save['image'] = image + + print("{0}/{1}: {2} is going to be added to the mentions collection." + .format(item_index + 1, n_items, item['key'])) + items_to_save.append(to_save) + + if len(items_to_save) > 0: + print("Items to save: %s" % len(items_to_save)) + resp = requests.post( + backend_url + '/mention?on_conflict=zotero_key', + json=items_to_save, + headers={'Authorization': 'Bearer %s' % jwt_token, 'Prefer': 'resolution=merge-duplicates'} + ) + resp.raise_for_status() + + if by_key: + ids_to_update_scraped_at = ",".join([k["id"] for k in keys_data]) + requests.patch( + backend_url + '/mention?id=in.(' + ids_to_update_scraped_at + ')', + json={'scraped_at': str(datetime.now())}, + headers={'Authorization': 'Bearer %s' % jwt_token} + ) + +if __name__ == "__main__": + print("Start scraping from Zotero") + backend_url = os.environ.get('POSTGREST_URL') + number_mentions_to_scrape = os.environ.get('MAX_REQUESTS_GITHUB', default='6') + jwt_secret = os.environ.get('PGRST_JWT_SECRET') + jwt_token = jwt.encode({"role": "rsd_admin", "exp": datetime.now() + timedelta(minutes = 10)}, jwt_secret, algorithm="HS256") + zotero_library = os.environ.get('ZOTERO_LIBRARY') + zotero_api_token = os.environ.get('ZOTERO_API_TOKEN') + get_mentions() + print("Done scraping from Zotero") diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/BibtexParserTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/BibtexParserTest.java new file mode 100644 index 0000000..13e74c2 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/BibtexParserTest.java @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper; + +import nl.esciencecenter.rsd.scraper.doi.ExternalMentionRecord; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Map; + +class BibtexParserTest { + + @Test + void givenValidBibtex_whenParsing_thenParsedCorrectly() throws InvalidInputException, IOException { + String validBibtex = """ + @misc{Brughmans2018, + author = {Brughmans, Tom}, + title = {{Network structures and assembling code in Netlogo, Tutorial}}, + url = {https://archaeologicalnetworks.wordpress.com/resources/#structures}, + year = {2018} + } + @article{Brughmans2016c, + author = {Brughmans, Tom and Poblome, Jeroen}, + doi = {10.15184/aqy.2016.35}, + file = {:Users/au656977/Library/Application Support/Mendeley Desktop/Downloaded/Brughmans, Poblome - 2016 - Roman bazaar or market economy Explaining tableware distributions through computational modelling.pdf:pdf}, + journal = {Antiquity}, + mendeley-groups = {Brughmans,PNAS,old/ERC_StG2019_B2,old/ERC_StG2019_B1,Roman-simulations,TO ADD,TRAJ manifesto,ERCStG2021_B2}, + number = {350}, + pages = {393--408}, + title = {{Roman bazaar or market economy? Explaining tableware distributions through computational modelling}}, + volume = {90}, + year = {2016} + } + @article{Brughmans2016d, + author = {Brughmans, Tom and Poblome, Jeroen}, + file = {:Users/au656977/Library/Application Support/Mendeley Desktop/Downloaded/Brughmans, Poblome - 2016 - MERCURY an agent-based model of tableware trade in the Roman East.pdf:pdf}, + journal = {Journal of Artificial Societies and Social Simulation}, + mendeley-groups = {Brughmans,old/ERC_StG2019_B2,Roman-simulations,TO ADD,TRAJ manifesto}, + number = {1}, + pages = {http://jasss.soc.surrey.ac.uk/19/1/3.html}, + title = {{MERCURY: an agent-based model of tableware trade in the Roman East}}, + volume = {19}, + year = {2016} + } + """; + + Map result = BibtexParser.parse(new StringReader(validBibtex)); + + Assertions.assertEquals(3, result.size()); + Assertions.assertEquals("Brughmans, Tom", result.get("Brughmans2018").authors()); + Assertions.assertEquals("Network structures and assembling code in Netlogo, Tutorial", result.get("Brughmans2018") + .title()); + Assertions.assertEquals(2018, result.get("Brughmans2018").publicationYear()); + } + + @Test + void givenValidBibtex_whenParsing_thenParsedCorrectly2() throws InvalidInputException, IOException { + String validBibtex = """ + @book{romanowska_agent-based_2021, + edition = {Electronic}, + title = {Agent-{Based} {Modeling} for {Archaeology}}, + isbn = {978-1-947864-38-2}, + url = {https://www.sfipress.org/books/agent-based-modeling-archaeology}, + abstract = {To fully understand not only the past, but also the trajectories, of human societies, we need a more dynamic view of human social systems. Agent-based modeling (ABM), which can create fine-scale models of behavior over time and space, may reveal important, general patterns of human activity. Agent-Based Modeling for Archaeology is the first ABM textbook designed for researchers studying the human past. Appropriate for scholars from archaeology, the digital humanities, and other social sciences, this book offers novices and more experienced ABM researchers a modular approach to learning ABM and using it effectively. Readers will find the necessary background, discussion of modeling techniques and traps, references, and algorithms to use ABM in their own work. They will also find engaging examples of how other scholars have applied ABM, ranging from the study of the intercontinental migration pathways of early hominins, to the weather–crop–population cycles of the American Southwest, to the trade networks of Ancient Rome. This textbook provides the foundations needed to simulate the complexity of past human societies, offering researchers a richer understanding of the past—and likely future—of our species.}, + urldate = {2022-02-01}, + publisher = {SFI Press}, + author = {Romanowska, Iza}, + month = aug, + year = {2021}, + doi = {10.37911/9781947864382}, + }"""; + + Map result = BibtexParser.parse(new StringReader(validBibtex)); + + Assertions.assertEquals(1, result.size()); + Assertions.assertEquals("Romanowska, Iza", result.get("romanowska_agent-based_2021").authors()); + Assertions.assertEquals("Agent-Based Modeling for Archaeology", result.get("romanowska_agent-based_2021") + .title()); + Assertions.assertEquals("10.37911/9781947864382", result.get("romanowska_agent-based_2021").doi().toString()); + Assertions.assertEquals(2021, result.get("romanowska_agent-based_2021").publicationYear()); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsIT.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsIT.java new file mode 100644 index 0000000..864378f --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsIT.java @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +class UtilsIT { + + private final String gitHubApiUrl = "https://api.github.com"; + private final String gitHubRepository = "research-software-directory/RSD-as-a-service"; + private final String gitHubUri = gitHubApiUrl + "/repos/" + gitHubRepository; + private final String dbUrl = Config.backendBaseUrl(); + + @Disabled + @Test + void getWithoutToken() { + final String jsonResponse = Assertions.assertDoesNotThrow(() -> Utils.get(gitHubUri)); + Assertions.assertTrue(jsonResponse.startsWith("{")); + } + + @Disabled + @Test + void getWrongUri() { + final String wrongUri = gitHubApiUrl + "/repos/research-software-directory/wrongRepo"; + Exception thrown = Assertions.assertThrows( + Exception.class, () -> Utils.get(wrongUri) + ); + Assertions.assertTrue(thrown.getMessage().startsWith("Error fetching data")); + } + + @Disabled + @Test + void getAsAdmin() { + String table = "repository_url"; + String value = Utils.getAsAdmin(dbUrl + "/" + table); + Assertions.assertTrue(value.startsWith("[{\"software\":")); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsTest.java new file mode 100644 index 0000000..2d5be15 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsTest.java @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2022 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class UtilsTest { + + @Test + void urlEncode() { + Assertions.assertEquals("one%2Ftwo", Utils.urlEncode("one/two")); + } + + @Test + void testCreatePatchUri() { + String baseUri = "http://localhost/api/v1"; + String tableName = "table_name"; + String primaryKey = "d8718d90-a3df-4714-864d-40786223e462"; + String primaryKeyName = "id"; + + String expectedUri = "http://localhost/api/v1/table_name?id=eq.d8718d90-a3df-4714-864d-40786223e462"; + String actualUri = Utils.createPatchUri(baseUri, tableName, primaryKey, primaryKeyName); + + Assertions.assertEquals(expectedUri, actualUri); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepositoryTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepositoryTest.java new file mode 100644 index 0000000..9a814de --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepositoryTest.java @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collection; +import java.util.List; + +class DataciteMentionRepositoryTest { + + @Test + void givenCollectionOfStrings_whenJoining_thenCorrectStringReturned() { + Doi doi1 = Doi.fromString("10.000/1"); + Doi doi2 = Doi.fromString("10.2/2"); + Doi doi3 = Doi.fromString("10.3/abc-def"); + Collection strings = List.of(doi1, doi2, doi3); + + String joinedString = DataciteMentionRepository.joinDoisForGraphqlQuery(strings); + + String expected = "\"%s\",\"%s\",\"%s\"".formatted(doi1, doi2, doi3); + Assertions.assertEquals(expected, joinedString); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DoiTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DoiTest.java new file mode 100644 index 0000000..58ff894 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DoiTest.java @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class DoiTest { + + @ParameterizedTest + @ValueSource(strings = { + "10.2533/chimia.2024.525", + "10.1017/9781108881425", + "10.3390/photonics11070630", + "10.1093/gigascience/giad048", + "10.1007/978-3-030-83508-8_2", + "10.22541/essoar.171500959.99365288/v1", + "10.1016/j.eswa.2023.120561", + }) + void givenValidDoi_whenInstanceCreated_thenNoExceptionThrown(String validDoi) { + Doi doi = Assertions.assertDoesNotThrow(() -> Doi.fromString(validDoi)); + Assertions.assertNotNull(doi); + } + + @ParameterizedTest + @ValueSource(strings = { + "10.2533", + "10.2533/", + "https://doi.org/10.2533/chimia.2024.525", + "10.3390/photonics 11070630", + "10.3390/photonics11070630 ", + "11.1016/j.eswa.2023.120561", + "", + }) + void givenInValidDoi_whenCreatingInstance_thenExceptionThrown(String invalidDoi) { + Assertions.assertThrows(RuntimeException.class, () -> Doi.fromString(invalidDoi)); + } + + @Test + void givenTwoValidDoisThatOnlyDifferInCase_whenComparing_thenTheyAreEqual() { + String upperCaseDoi = "10.2533/chimia.2024.525"; + String lowerCaseDoi = "10.2533/CHIMIA.2024.525"; + + Doi doi1 = Doi.fromString(upperCaseDoi); + Doi doi2 = Doi.fromString(lowerCaseDoi); + + Assertions.assertEquals(doi1, doi2); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/MainMentionsTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/MainMentionsTest.java new file mode 100644 index 0000000..3eb6e1b --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/MainMentionsTest.java @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +class MainMentionsTest { + + + @Test + void givenValidDoiSourceData_whenParsing_thenMapReturned() { + // editorconfig-checker-disable + String validDoiSourceData = """ + [ + { + "DOI": "10.5240/B1FA-0EEC-C316-3316-3A73-L", + "RA": "EIDR" + }, + { + "DOI": "notADoi", + "status": "Invalid DOI" + }, + { + "DOI": "10.5281/zenodo.1436372", + "RA": "DataCite" + }, + { + "DOI": "10.5281/zenodo.2633819", + "RA": "DataCite" + }, + { + "DOI": "10.5281/zenodo.5825192", + "RA": "DataCite" + }, + { + "DOI": "10.35802/218300", + "RA": "Crossref" + } + ]"""; + // editorconfig-checker-enable + + Map doiToSource = MainMentions.parseJsonDoiSources(validDoiSourceData); + Assertions.assertEquals(6, doiToSource.size()); + Assertions.assertEquals("EIDR", doiToSource.get("10.5240/B1FA-0EEC-C316-3316-3A73-L")); + Assertions.assertEquals("Invalid DOI", doiToSource.get("notADoi")); + Assertions.assertEquals("DataCite", doiToSource.get("10.5281/zenodo.1436372")); + Assertions.assertEquals("DataCite", doiToSource.get("10.5281/zenodo.2633819")); + Assertions.assertEquals("DataCite", doiToSource.get("10.5281/zenodo.5825192")); + Assertions.assertEquals("Crossref", doiToSource.get("10.35802/218300")); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnectorTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnectorTest.java new file mode 100644 index 0000000..fab34b5 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnectorTest.java @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.util.Collection; +import java.util.Collections; + +class OpenAlexConnectorTest { + + @Test + void givenLocationWithBackSlashes_whenExtractedAsLocation_thenSlashesUrlEncoded() { + JsonArray array = new JsonArray(); + // Example of the structure: https://api.openalex.org/works/https://doi.org/10.2777/28598 + JsonObject location = new JsonObject(); + location.addProperty("landing_page_url", "https://www.example.com/path\\with\\slash"); + array.add(location); + + URI result = OpenAlexConnector.extractUrlFromLocation(array); + + Assertions.assertNotNull(result); + Assertions.assertEquals("https://www.example.com/path%5Cwith%5Cslash", result.toString()); + } + + @Test + void givenEmptyCollection_whenGettingData_thenEmptyCollectionReturned() { + OpenAlexConnector openAlexConnector = new OpenAlexConnector(); + + Collection doiMentions = Assertions.assertDoesNotThrow(() -> openAlexConnector.mentionDataByDois(Collections.emptyList(), null)); + Assertions.assertTrue(doiMentions.isEmpty()); + + Collection openalexMentions = Assertions.assertDoesNotThrow(() -> openAlexConnector.mentionDataByOpenalexIds(Collections.emptyList(), null)); + Assertions.assertTrue(openalexMentions.isEmpty()); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenalexIdTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenalexIdTest.java new file mode 100644 index 0000000..0b20e9a --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenalexIdTest.java @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class OpenalexIdTest { + + @ParameterizedTest + @ValueSource(strings = { + "https://openalex.org/W3160330321", + "https://openalex.org/w3160330321", + "https://openalex.org/W152867311", + }) + void givenValidOpenalexId_whenInstanceCreated_thenNoExceptionThrown(String validId) { + OpenalexId openalexId = Assertions.assertDoesNotThrow(() -> OpenalexId.fromString(validId)); + Assertions.assertNotNull(openalexId); + } + + @ParameterizedTest + @ValueSource(strings = { + "http://openalex.org/W3160330321", + "https://openalex.org/3160330321", + "https://openalex.org/W3160330321/", + "https://openalex.org/works/W3160330321", + "W3160330321", + "", + }) + void givenInValidOpenalexId_whenCreatingInstance_thenExceptionThrown(String invalidId) { + Assertions.assertThrows(RuntimeException.class, () -> OpenalexId.fromString(invalidId)); + } + + @Test + void givenTwoValidIdsThatOnlyDifferInCase_whenComparing_thenTheyAreEqual() { + String upperCaseId = "https://openalex.org/W3160330321"; + String lowerCaseId = "https://openalex.org/w3160330321"; + + OpenalexId openalexId1 = OpenalexId.fromString(upperCaseId); + OpenalexId openalexId2 = OpenalexId.fromString(lowerCaseId); + + Assertions.assertEquals(openalexId1, openalexId2); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeekTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeekTest.java new file mode 100644 index 0000000..56788b3 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeekTest.java @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2025 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializer; +import com.google.gson.reflect.TypeToken; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.time.Instant; +import java.time.Period; +import java.util.Map; + +class CommitsPerWeekTest { + + @Test + void givenInstance_whenValidOperations_thenCorrectResults() { + CommitsPerWeek commitsPerWeek = new CommitsPerWeek(); + Map underlyingMap; + + Instant sundayMidnight1 = Instant.ofEpochSecond(1670716800); + commitsPerWeek.addCommits(sundayMidnight1, 10); + + underlyingMap = commitsPerWeek.getData(); + Assertions.assertEquals(1, underlyingMap.size()); + Assertions.assertTrue(underlyingMap.containsKey(sundayMidnight1)); + Assertions.assertEquals(10, underlyingMap.get(sundayMidnight1)); + + + commitsPerWeek.addCommits(sundayMidnight1, 20); + + underlyingMap = commitsPerWeek.getData(); + Assertions.assertEquals(1, underlyingMap.size()); + Assertions.assertTrue(underlyingMap.containsKey(sundayMidnight1)); + Assertions.assertEquals(30, underlyingMap.get(sundayMidnight1)); + + + Instant smallTimeAfterSundayMidnight1 = sundayMidnight1.plus(Duration.ofDays(3)) + .plus(Duration.ofSeconds(12345)); + commitsPerWeek.addCommits(smallTimeAfterSundayMidnight1, 10); + + underlyingMap = commitsPerWeek.getData(); + Assertions.assertEquals(1, underlyingMap.size()); + Assertions.assertTrue(underlyingMap.containsKey(sundayMidnight1)); + Assertions.assertEquals(40, underlyingMap.get(sundayMidnight1)); + + + Instant sundayMidnight2 = sundayMidnight1.plus(Period.ofWeeks(5)); + commitsPerWeek.addCommits(sundayMidnight2, 5); + + underlyingMap = commitsPerWeek.getData(); + Assertions.assertEquals(2, underlyingMap.size()); + Assertions.assertTrue(underlyingMap.containsKey(sundayMidnight2)); + Assertions.assertEquals(5, underlyingMap.get(sundayMidnight2)); + Assertions.assertEquals(40, underlyingMap.get(sundayMidnight1)); + + + commitsPerWeek.addMissingZeros(); + underlyingMap = commitsPerWeek.getData(); + Assertions.assertEquals(6, underlyingMap.size()); + Assertions.assertEquals(0, underlyingMap.get(sundayMidnight1.plus(Period.ofWeeks(1)))); + Assertions.assertEquals(0, underlyingMap.get(sundayMidnight1.plus(Period.ofWeeks(2)))); + Assertions.assertEquals(0, underlyingMap.get(sundayMidnight1.plus(Period.ofWeeks(3)))); + Assertions.assertEquals(0, underlyingMap.get(sundayMidnight1.plus(Period.ofWeeks(4)))); + Assertions.assertEquals(40, underlyingMap.get(sundayMidnight1)); + Assertions.assertEquals(5, underlyingMap.get(sundayMidnight2)); + + } + + @Test + void givenInstance_whenValidOperations_thenCorrectJsonProduced() { + Gson gson = new GsonBuilder() + .registerTypeAdapter(Instant.class, (JsonDeserializer) (json, typeOfT, context) -> Instant.ofEpochSecond(Long.parseLong(json.getAsString()))) + .create(); + TypeToken> mapTypeToken = new TypeToken<>() { + }; + CommitsPerWeek commitsPerWeek = new CommitsPerWeek(); + + Map shouldBeEmptyMap = gson.fromJson(commitsPerWeek.toJson(), mapTypeToken.getType()); + Assertions.assertTrue(shouldBeEmptyMap.isEmpty()); + + Instant sundayMidnight1 = Instant.ofEpochSecond(1670716800); + commitsPerWeek.addCommits(sundayMidnight1, 10); + commitsPerWeek.addCommits(sundayMidnight1, 20); + + Instant smallTimeAfterSundayMidnight1 = sundayMidnight1.plus(Duration.ofDays(3)) + .plus(Duration.ofSeconds(12345)); + commitsPerWeek.addCommits(smallTimeAfterSundayMidnight1, 10); + + Instant sundayMidnight2 = sundayMidnight1.plus(Period.ofWeeks(5)); + commitsPerWeek.addCommits(sundayMidnight2, 5); + + + Map dataFromJson = gson.fromJson(commitsPerWeek.toJson(), mapTypeToken.getType()); + Assertions.assertEquals(2, dataFromJson.size()); + Assertions.assertEquals(40, dataFromJson.get(sundayMidnight1)); + Assertions.assertEquals(5, dataFromJson.get(sundayMidnight2)); + + commitsPerWeek.addMissingZeros(); + dataFromJson = gson.fromJson(commitsPerWeek.toJson(), new TypeToken>() { + }.getType()); + Assertions.assertEquals(6, dataFromJson.size()); + Assertions.assertEquals(0, dataFromJson.get(sundayMidnight1.plus(Period.ofWeeks(1)))); + Assertions.assertEquals(0, dataFromJson.get(sundayMidnight1.plus(Period.ofWeeks(2)))); + Assertions.assertEquals(0, dataFromJson.get(sundayMidnight1.plus(Period.ofWeeks(3)))); + Assertions.assertEquals(0, dataFromJson.get(sundayMidnight1.plus(Period.ofWeeks(4)))); + Assertions.assertEquals(40, dataFromJson.get(sundayMidnight1)); + Assertions.assertEquals(5, dataFromJson.get(sundayMidnight2)); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GithubScraperIT.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GithubScraperIT.java new file mode 100644 index 0000000..12f1275 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GithubScraperIT.java @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +class GithubScraperIT { + + private final String githubUrlPrefix = "https://github.com/"; + private final String repo = "research-software-directory/RSD-as-a-service"; + private final String repoEmpty = "cmeessen/empty"; + private final String repoNonEx = "research-software-directory/does-not-exist"; + + private final GithubScraper githubScraper = GithubScraper.create(githubUrlPrefix + repo).get(); + private final GithubScraper githubScraperEmpty = GithubScraper.create(githubUrlPrefix + repoEmpty).get(); + private final GithubScraper githubScraperNonEx = GithubScraper.create(githubUrlPrefix + repoNonEx).get(); + + @Disabled + @Test + void languages() { + final String languages = Assertions.assertDoesNotThrow(() -> githubScraper.languages()); + Assertions.assertTrue(languages.startsWith("{")); + Assertions.assertTrue(languages.endsWith("}")); + Assertions.assertTrue(languages.contains("Java")); + } + + @Disabled + @Test + void license() { + String license = Assertions.assertDoesNotThrow(() -> githubScraper.basicData().license()); + Assertions.assertEquals("Apache-2.0", license); + } + + @Disabled + @Test + void contributions() { + final CommitsPerWeek contributions = Assertions.assertDoesNotThrow(githubScraper::contributions); + // Assertions.assertTrue(contributions.startsWith("[{\"total")); + } + + @Disabled + @Test + void contributionsEmpty() { + final CommitsPerWeek contributionsEmpty = Assertions.assertDoesNotThrow(githubScraperEmpty::contributions); + // Assertions.assertTrue("[]", contributionsEmpty); + } + + @Disabled + @Test + void contributionsNonEx() { + final CommitsPerWeek contributionsNonEx = Assertions.assertDoesNotThrow(githubScraperNonEx::contributions); + // Assertions.assertNull(contributionsNonEx); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GithubScraperTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GithubScraperTest.java new file mode 100644 index 0000000..52af1d5 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GithubScraperTest.java @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; + +class GithubScraperTest { + + private final String githubUrlPrefix = "https://github.com/"; + private final String repo = "research-software-directory/RSD-as-a-service"; + private final String repoGit = "research-software-directory/RSD-as-a-service.git"; + private final String repoEmpty = "cmeessen/empty"; + private final String repoNonEx = "research-software-directory/does-not-exist"; + + @Test + void givenListWithLastPageHeader_whenParsing_thenCorrectPageReturned() { + List singleLinkList = List.of("; rel=\"next\", ; rel=\"last\""); + + String[] lastPageData = GithubScraper.lastPageFromLinkHeader(singleLinkList); + Assertions.assertEquals(2, lastPageData.length); + Assertions.assertEquals("https://api.github.com/repositories/413814951/contributors?per_page=1&page=9", lastPageData[0]); + Assertions.assertEquals("9", lastPageData[1]); + } + + @Test + void givenListWithoutLastPage_whenParsing_thenExceptionThrown() { + List singleLinkList = List.of("invalid"); + + Assertions.assertThrows(RuntimeException.class, () -> GithubScraper.lastPageFromLinkHeader(singleLinkList)); + } + + @Test + void givenValidGithubUrl_whenCreatingScraper_thenNonEmptyScraperReturned() { + Optional scraper1 = GithubScraper.create(githubUrlPrefix + repo); + Assertions.assertTrue(scraper1.isPresent()); + + Optional scraper2 = GithubScraper.create(githubUrlPrefix + repoEmpty); + Assertions.assertTrue(scraper2.isPresent()); + + Optional scraper3 = GithubScraper.create(githubUrlPrefix + repoNonEx + "/"); + Assertions.assertTrue(scraper3.isPresent()); + } + + @Test + void givenInValidGithubUrl_whenCreatingScraper_thenEmptyScraperReturned() { + Optional scraper1 = GithubScraper.create(githubUrlPrefix + repo + "/issues"); + Assertions.assertTrue(scraper1.isEmpty()); + + Optional scraper2 = GithubScraper.create(githubUrlPrefix + repoEmpty + "/tree/main"); + Assertions.assertTrue(scraper2.isEmpty()); + + Optional scraper3 = GithubScraper.create(githubUrlPrefix + "org-only/"); + Assertions.assertTrue(scraper3.isEmpty()); + } + + @Test + void givenGitRepoUrl_whenCreatingScraper_thenRemoveGitSuffix() { + Optional scraper = GithubScraper.create(githubUrlPrefix + repoGit); + Assertions.assertEquals(repo, scraper.get().organisation + "/" + scraper.get().repo); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GitlabScraperIT.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GitlabScraperIT.java new file mode 100644 index 0000000..f3e4b5f --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GitlabScraperIT.java @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + + +class GitlabScraperIT { + private final String baseApiUrl = "https://git.gfz-potsdam.de/api"; + private final String repo = "swc-bb/swc-templates/swc-l/python"; + private final GitlabScraper scraper = new GitlabScraper(baseApiUrl, repo); + + @Disabled + @Test + void languages() { + final String languages = Assertions.assertDoesNotThrow(scraper::languages); + Assertions.assertTrue(languages.startsWith("{")); + Assertions.assertTrue(languages.endsWith("}")); + Assertions.assertTrue(languages.contains("Python")); + } + + @Disabled + @Test + void license() { + String license = Assertions.assertDoesNotThrow(() -> scraper.basicData().license()); + Assertions.assertEquals("MIT License", license); + } + + @Disabled + @Test + void licenseDoesNotExist() { + // unlicensed projects should return null + // we need to find a suitable project or create a mocked interface + String license = Assertions.assertDoesNotThrow(() -> scraper.basicData().license()); + Assertions.assertNull(license); + } + + @Disabled + @Test + void contributions() { + final CommitsPerWeek contributions = Assertions.assertDoesNotThrow(scraper::contributions); + // Assertions.assertTrue(contributions.contains("360071ec98b3c0d38ac8e7f0abacc05098c90311")); + // Assertions.assertTrue(contributions.contains("2019-10-24T16:40:44.000+02:00")); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnectorTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnectorTest.java new file mode 100644 index 0000000..4a53d3c --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnectorTest.java @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collection; +import java.util.UUID; + +class PostgrestConnectorTest { + + @Test + void givenEmptyJsonArray_whenParsingJsonData_thenEmptyCollectionReturned() { + String emptyArrayJson = "[]"; + + Collection result = PostgrestConnector.parseBasicJsonData(emptyArrayJson); + + Assertions.assertTrue(result.isEmpty()); + } + + @Test + void givenValidJson_whenParsingJsonData_thenRecordWithDataReturned() { + String arrayWithSingleValidObjectJson = """ + [ + { + "software": "3a07a021-743e-4adf-a2d9-3c85075fe9cf", + "url": "https://www.example.com" + } + ]"""; + + Collection result = PostgrestConnector.parseBasicJsonData(arrayWithSingleValidObjectJson); + + Assertions.assertEquals(1, result.size()); + BasicRepositoryData dataToInspect = result.stream().findFirst().get(); + Assertions.assertEquals(UUID.fromString("3a07a021-743e-4adf-a2d9-3c85075fe9cf"), dataToInspect.software()); + Assertions.assertEquals("https://www.example.com", dataToInspect.url()); + } + + @Test + void givenJsonWithInvalidUuid_whenParsingJsonData_thenExceptionThrown() { + String invalidUuidJson = """ + [ + { + "software": "not-a-UUID", + "url": "https://www.example.com" + } + ]"""; + + Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> PostgrestConnector.parseBasicJsonData(invalidUuidJson)); + } + + @Test + void givenInvalidJson_whenParsingJsonData_thenExceptionThrown() { +// missing comma after software UUID + String invalidJson = """ + [ + { + "software": "3a07a021-743e-4adf-a2d9-3c85075fe9cf" + "url": "https://www.example.com" + } + ]"""; + + Assertions.assertThrows(RuntimeException.class, () -> PostgrestConnector.parseBasicJsonData(invalidJson)); + } + + @Test + void givenValidJsonWithMissingFields_whenParsingJsonData_thenExceptionThrown() { + String missingSoftwareJson = """ + [ + { + "url": "https://www.example.com" + } + ]"""; + + Assertions.assertThrows(RuntimeException.class, () -> PostgrestConnector.parseBasicJsonData(missingSoftwareJson)); + + String missingUrlJson = """ + [ + { + "software": "3a07a021-743e-4adf-a2d9-3c85075fe9cf" + } + ]"""; + + Assertions.assertThrows(RuntimeException.class, () -> PostgrestConnector.parseBasicJsonData(missingUrlJson)); + } + + @Test + void givenValidJsonWithNullFields_whenParsingJsonData_thenExceptionThrown() { + String nullSoftwareJson = """ + [ + { + "software": null, + "url": "https://www.example.com" + } + ]"""; + + Assertions.assertThrows(RuntimeException.class, () -> PostgrestConnector.parseBasicJsonData(nullSoftwareJson)); + + String nullUrlJson = """ + [ + { + "software": "3a07a021-743e-4adf-a2d9-3c85075fe9cf", + "url": null + } + ]"""; + + Assertions.assertThrows(RuntimeException.class, () -> PostgrestConnector.parseBasicJsonData(nullUrlJson)); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepositoryTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepositoryTest.java new file mode 100644 index 0000000..bc32ac4 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepositoryTest.java @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.license; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +class GitHubSpdxLicenseRepositoryTest { + + @Test + void parseLicensesJson() { + // editorconfig-checker-disable + String json = """ + { + "licenseListVersion": "779bbe0", + "licenses": [ + { + "reference": "https://spdx.org/licenses/0BSD.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/0BSD.json", + "referenceNumber": 16, + "name": "BSD Zero Clause License", + "licenseId": "0BSD", + "seeAlso": [ + "http://landley.net/toybox/license.html", + "https://opensource.org/licenses/0BSD" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/MIT.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/MIT.json", + "referenceNumber": 535, + "name": "MIT License", + "licenseId": "MIT", + "seeAlso": [ + "https://opensource.org/license/mit/" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, + { + "reference": "https://spdx.org/licenses/CC-BY-4.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-4.0.json", + "referenceNumber": 578, + "name": "Creative Commons Attribution 4.0 International", + "licenseId": "CC-BY-4.0", + "seeAlso": [ + "https://creativecommons.org/licenses/by/4.0/legalcode" + ], + "isOsiApproved": false, + "isFsfLibre": true + } + ], + "releaseDate": "2024-10-10" + }"""; + // editorconfig-checker-enable + + Map licenseMap = Assertions.assertDoesNotThrow(() -> GitHubSpdxLicenseRepository.parseLicensesJson(json)); + Assertions.assertEquals(3, licenseMap.size()); + Assertions.assertTrue(licenseMap.containsKey("0BSD")); + Assertions.assertTrue(licenseMap.containsKey("MIT")); + Assertions.assertTrue(licenseMap.containsKey("CC-BY-4.0")); + + SpdxLicense bsdZero = licenseMap.get("0BSD"); + Assertions.assertEquals("0BSD", bsdZero.licenseId()); + Assertions.assertEquals("https://spdx.org/licenses/0BSD.html", bsdZero.reference()); + Assertions.assertEquals("BSD Zero Clause License", bsdZero.name()); + + SpdxLicense mit = licenseMap.get("MIT"); + Assertions.assertEquals("MIT", mit.licenseId()); + Assertions.assertEquals("https://spdx.org/licenses/MIT.html", mit.reference()); + Assertions.assertEquals("MIT License", mit.name()); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntryTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntryTest.java new file mode 100644 index 0000000..1f4c7ac --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntryTest.java @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.nassa; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.StringReader; + +class NassaSoftwareEntryTest { + + @Test + void givenValidNassaYaml_whenParsing_thenSoftwareEntryReturned() { + // editorconfig-checker-disable + String validYaml = """ + id: 2022-Verhagen-001 + nassaVersion: 0.5.0 + moduleType: Algorithm + title: Determine fertility rates for use in a demographic simulation. + moduleVersion: 1.0.1 + contributors: + - name: Verhagen, Philip + roles: [ "Author", "Copyright Holder", "Creator" ] + email: j.w.h.p.verhagen@vu.nl + orcid: 0000-0001-8166-122X + lastUpdateDate: 2022-12-02 + description: > + This NetLogo code snippet provides fertility estimates of females for use in demographic simulations. The figures used are based on two different sources, and will be assigned per 5-year age cohort. + relatedModules:\s + references: + moduleReferences: [ Bagnall-Frier-1994, Coale-Trussell-1974, Coale-Trussell-1978, Henry-1961, Verhagen-2022 ] + domainKeywords: + regions:\s + periods:\s + - Roman period + - Modern period + subjects:\s + - demography + - fertility + modellingKeywords: + - parameter initialisation + programmingKeywords: + - Functional + - Object-Oriented + implementations: + - language: NetLogo + softwareDependencies:\s + - NetLogo 6.2.0 + docsDir: documentation/ + license: MIT + """; + // editorconfig-checker-enable + + NassaSoftwareEntry entry = NassaSoftwareEntry.fromYaml(new StringReader(validYaml), "https://github.com/Archaeology-ABM/NASSA-modules/tree/main/2022-Verhagen-001", null); + + Assertions.assertNotNull(entry); + Assertions.assertEquals("Determine fertility rates for use in a demographic simulation.", entry.title); + Assertions.assertEquals("1.0.1", entry.moduleVersion); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraperTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraperTest.java new file mode 100644 index 0000000..2c527c3 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraperTest.java @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.NullSource; +import org.junit.jupiter.params.provider.ValueSource; + +class CranScraperTest { + + @ParameterizedTest + @CsvSource({ + "https://CRAN.R-project.org/package=TIMP,TIMP", + "https://CRAN.R-project.org/package=TIMP/,TIMP", + "https://cran.r-project.org/package=splithalfr,splithalfr", + "https://cran.r-project.org/web/packages/GGIR,GGIR", + "https://cran.r-project.org/web/packages/GGIR/,GGIR", + "https://cran.r-project.org/web/packages/GGIR/index.html,GGIR", + "https://cran.r-project.org/web/packages/GGIR/index.html/,GGIR" + }) + void givenValidCranUrl_whenCallingConstructor_thenNoExceptionThrownAndPackageNamesCorrect(String url, String expectedPackageName) { + CranScraper cranScraper = Assertions.assertDoesNotThrow(() -> new CranScraper(url)); + Assertions.assertEquals(expectedPackageName, cranScraper.packageName); + } + + @ParameterizedTest + @ValueSource(strings = { + "https://cran.r-project.org/web/packages/", + "https://cran.r-project.org/web/packages/GG IR", + "https://cran.r-project.org/web/packages/GGIR ", + "https://cran.r-project.org/web/packages/GGIR /", + "https://CRAN.R-project.org/package=", + "https://CRAN.R-project.org/package=TIMP ", + "https://CRAN.R-project.org/package=TI MP ", + "https://CRAN.R-project.org/package=TIMP /", + "https://www.example.com", + "" + }) + void givenInvalidCranUrl_whenCallingConstructor_thenExceptionThrown(String url) { + Assertions.assertThrowsExactly(RuntimeException.class, () -> new CranScraper(url)); + } + + @ParameterizedTest + @NullSource + void givenNullUrl_whenCallingConstructor_thenExceptionThrown(String url) { + Assertions.assertThrowsExactly(NullPointerException.class, () -> new CranScraper(url)); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraperTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraperTest.java new file mode 100644 index 0000000..b0dacb0 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraperTest.java @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.NullSource; + +class CratesScraperTest { + + @ParameterizedTest + @CsvSource({ + "https://crates.io/crates/tokio,tokio", + "https://crates.io/crates/tokio/,tokio", + }) + void givenValidCratesUrl_whenCallingConstructor_thenNoExceptionThrownAndPackageNamesCorrect( + String url, + String expectedPackageName + ) { + CratesScraper cratesScraper = Assertions.assertDoesNotThrow(() -> new CratesScraper(url)); + Assertions.assertEquals(expectedPackageName, cratesScraper.packageName); + } + + @ParameterizedTest + @NullSource + void givenNullUrl_whenCallingConstructor_thenExceptionThrown(String url) { + Assertions.assertThrowsExactly(NullPointerException.class, () -> new CratesScraper(url)); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraperTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraperTest.java new file mode 100644 index 0000000..452c671 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraperTest.java @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.NullSource; + +class GoScraperTest { + + @ParameterizedTest + @CsvSource({ + "https://pkg.go.dev/github.com/gin-gonic/gin,github.com/gin-gonic/gin", + "https://pkg.go.dev/github.com/gin-gonic/gin/,github.com/gin-gonic/gin", + "https://pkg.go.dev/google.golang.org/grpc,google.golang.org/grpc", + "https://pkg.go.dev/google.golang.org/grpc/,google.golang.org/grpc", + }) + void givenValidGoUrl_whenCallingConstructor_thenNoExceptionThrownAndPackageNameCorrect( + String url, + String expectedPackageName + ) { + GoScraper goScraper = Assertions.assertDoesNotThrow(() -> new GoScraper(url)); + Assertions.assertEquals(expectedPackageName, goScraper.packageName); + } + + @ParameterizedTest + @NullSource + void givenNullUrl_whenCallingConstructor_thenExceptionThrown(String url) { + Assertions.assertThrowsExactly(NullPointerException.class, () -> new GoScraper(url)); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraperTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraperTest.java new file mode 100644 index 0000000..2f7cf99 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraperTest.java @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.NullSource; +import org.junit.jupiter.params.provider.ValueSource; + +class MavenScraperTest { + + @ParameterizedTest + @CsvSource({ + "https://central.sonatype.com/artifact/org.openscience.cdk/cdk-bundle,org.openscience.cdk,cdk-bundle", + "https://central.sonatype.com/artifact/org.openscience.cdk/cdk-bundle/,org.openscience.cdk,cdk-bundle", + "https://mvnrepository.com/artifact/io.github.sanctuuary/APE,io.github.sanctuuary,APE", + "https://mvnrepository.com/artifact/io.github.sanctuuary/APE/,io.github.sanctuuary,APE", + }) + void givenValidMavenOrSonatypeUrl_whenCallingConstructor_thenNoExceptionThrownAndPackageNameCorrect( + String url, + String expectedGroupId, + String expectedArtifactID + ) { + MavenScraper mavenScraper = Assertions.assertDoesNotThrow(() -> new MavenScraper(url)); + Assertions.assertEquals(expectedGroupId, mavenScraper.groupId); + Assertions.assertEquals(expectedArtifactID, mavenScraper.artifactId); + } + + @ParameterizedTest + @ValueSource(strings = { + "https://central.sonatype.com/artifact", + "https://mvnrepository.com/artifact/", + "https://central.sonatype.com/artifact/org.openscience.cdk", + "https://mvnrepository.com/artifact/io.github.sanctuuary/", + "https://www.example.com", + "https://www.example.com/artifact/org.openscience.cdk/cdk-bundle", + "" + }) + void givenInvalidUrl_whenCallingConstructor_thenExceptionThrown(String url) { + Assertions.assertThrowsExactly(RuntimeException.class, () -> new MavenScraper(url)); + } + + @ParameterizedTest + @NullSource + void givenNullUrl_whenCallingConstructor_thenExceptionThrown(String url) { + Assertions.assertThrowsExactly(NullPointerException.class, () -> new MavenScraper(url)); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorIdTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorIdTest.java new file mode 100644 index 0000000..8d029c7 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorIdTest.java @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.ror; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.net.URI; + +class RorIdTest { + + @ParameterizedTest + @ValueSource(strings = { + "https://ror.org/14tsk2644", // ID should start with 0 + "https://ror.org/0ltsk2644", // character 'l' not allowed + "https://ror.org/04tsk26444", // ID longer than 9 characters + "https://ror.org/04tsk264", // // ID shorter than 9 characters + "ror.org/04tsk2644", // should start with https:// + "", + }) + void givenInvalidRorIds_whenTesting_thenFalseReturned(String url) { + Assertions.assertFalse(RorId.isValidRorUrl(url)); + } + + @ParameterizedTest + @ValueSource(strings = { + "https://ror.org/14tsk2644", // ID should start with 0 + "https://ror.org/0ltsk2644", // character 'l' not allowed + "https://ror.org/04tsk26444", // ID longer than 9 characters + "https://ror.org/04tsk264", // // ID shorter than 9 characters + "ror.org/04tsk2644", // should start with https:// + "", + }) + void givenInvalidRorIds_whenCreatingInstance_thenExceptionThrown(String url) { + Assertions.assertThrows(IllegalArgumentException.class, () -> RorId.fromUrlString(url)); + } + + @ParameterizedTest + @ValueSource(strings = { + "https://ror.org/04tsk2644", + "https://ror.org/05qghxh33", + "https://ror.org/01e6qks80", + "https://ror.org/00wjc7c48", + "https://ror.org/01swzsf04", + "https://ror.org/02jbv0t02", + "https://ror.org/05grdyy37", + "https://ror.org/05f950310", + "https://ror.org/04ke6ht85", + "https://ror.org/018dfmf50", + }) + void givenValidRorIds_whenTesting_thenTrueReturned(String url) { + Assertions.assertTrue(RorId.isValidRorUrl(url)); + } + + @ParameterizedTest + @ValueSource(strings = { + "https://ror.org/04tsk2644", + "https://ror.org/05qghxh33", + "https://ror.org/01e6qks80", + "https://ror.org/00wjc7c48", + "https://ror.org/01swzsf04", + "https://ror.org/02jbv0t02", + "https://ror.org/05grdyy37", + "https://ror.org/05f950310", + "https://ror.org/04ke6ht85", + "https://ror.org/018dfmf50", + }) + void givenValidRorIds_whenCreatingInstance_thenNoExceptionThrown(String url) { + RorId rorId = Assertions.assertDoesNotThrow(() -> RorId.fromUrlString(url)); + Assertions.assertNotNull(rorId); + } + + @Test + void givenValidRorId_whenGettingUrls_thenCorrectUrlsReturned() { + RorId rorId = RorId.fromUrlString("https://ror.org/04tsk2644"); + + Assertions.assertEquals(URI.create("https://ror.org/04tsk2644"), rorId.asUrl()); + Assertions.assertEquals(URI.create("https://api.ror.org/v1/organizations/04tsk2644"), rorId.asApiV1Url()); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnectorTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnectorTest.java new file mode 100644 index 0000000..83e2971 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnectorTest.java @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.ror; + +import org.junit.jupiter.api.Test; + +import java.util.Collection; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +class RorPostgrestConnectorTest { + + @Test + void testParseBasicJsonData() { + String data = "[{\"id\":\"52b13a36-334b-429a-9cbb-5215264b36d2\",\"parent\":null,\"primary_maintainer\":\"22f87e87-7fd9-4716-9dc6-e790372cac4c\",\"slug\":\"gfz\",\"name\":\"Helmholtz Centre Potsdam GFZ German Research Centre for Geosciences\",\"ror_id\":\"https://ror.org/04z8jg394\",\"website\":\"https://www.gfz-potsdam.de\",\"is_tenant\":true,\"created_at\":\"2022-07-26T19:34:55.173342+00:00\",\"updated_at\":\"2023-05-17T06:11:51.338233+00:00\",\"description\":\"# About GFZ Potsdam\n\nThe GFZ is Germany’s national research center for the solid Earth Sciences. Our mission is to deepen the knowledge of the dynamics of the solid Earth, and to develop solutions for grand challenges facing society. These challenges include anticipating the hazards arising from the Earth’s dynamic systems and mitigating the associated risks to society; securing our habitat under the pressure of global change; and supplying energy and mineral resources for a rapidly growing population in a sustainable manner and without harming the environment. \n\n## Research Software at GFZ\n\nTo learn more about the research software policy at GFZ, visit [gfz-potsdam.de/en/software](https://www.gfz-potsdam.de/en/software).\n\n\",\"logo_id\":\"da40b40113fdfb1d3ce40f58ecb784fe33ba40bb\",\"country\":null,\"city\":null,\"short_description\":null}]"; + + Collection result = RorPostgrestConnector.parseBasicJsonData(data); + assertEquals(1, result.size()); + + OrganisationDatabaseData organisation = result.iterator().next(); + assertEquals(UUID.fromString("52b13a36-334b-429a-9cbb-5215264b36d2"), organisation.id()); + assertEquals(RorId.fromUrlString("https://ror.org/04z8jg394"), organisation.rorId()); + assertNull(organisation.data()); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorScraperTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorScraperTest.java new file mode 100644 index 0000000..f6b1645 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorScraperTest.java @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.ror; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +class RorScraperTest { + + + @Test + void testLocations() { + String completeJsonResponse = "{\"id\":\"https://ror.org/04z8jg394\",\"name\":\"Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences\",\"email_address\":\"\",\"ip_addresses\":[],\"established\":1992,\"types\":[\"Facility\"],\"relationships\":[{\"label\":\"Helmholtz Association of German Research Centres\",\"type\":\"Parent\",\"id\":\"https://ror.org/0281dp749\"}],\"addresses\":[{\"lat\":52.39886,\"lng\":13.06566,\"state\":null,\"state_code\":null,\"city\":\"Potsdam\",\"geonames_city\":{\"id\":2852458,\"city\":\"Potsdam\",\"geonames_admin1\":{\"name\":\"Brandenburg\",\"id\":2945356,\"ascii_name\":\"Brandenburg\",\"code\":\"DE.11\"},\"geonames_admin2\":{\"name\":null,\"id\":null,\"ascii_name\":null,\"code\":\"DE.11.00\"},\"license\":{\"attribution\":\"Data from geonames.org under a CC-BY 3.0 license\",\"license\":\"http://creativecommons.org/licenses/by/3.0/\"},\"nuts_level1\":{\"name\":null,\"code\":null},\"nuts_level2\":{\"name\":null,\"code\":null},\"nuts_level3\":{\"name\":null,\"code\":null}},\"postcode\":null,\"primary\":false,\"line\":null,\"country_geonames_id\":2921044}],\"links\":[\"https://www.gfz-potsdam.de\"],\"aliases\":[],\"acronyms\":[\"GFZ\"],\"status\":\"active\",\"wikipedia_url\":\"https://en.wikipedia.org/wiki/GFZ_German_Research_Centre_for_Geosciences\",\"labels\":[{\"label\":\"Helmholtz-Zentrum Potsdam - Deutsches GeoForschungsZentrum GFZ\",\"iso639\":\"de\"}],\"country\":{\"country_name\":\"Germany\",\"country_code\":\"DE\"},\"external_ids\":{\"ISNI\":{\"preferred\":null,\"all\":[\"0000 0000 9195 2461\"]},\"FundRef\":{\"preferred\":\"501100010956\",\"all\":[\"501100010956\"]},\"Wikidata\":{\"preferred\":null,\"all\":[\"Q1205654\"]},\"GRID\":{\"preferred\":\"grid.23731.34\",\"all\":\"grid.23731.34\"}}}"; + RorData rorData = RorScraper.parseData(completeJsonResponse); + + assertEquals("Potsdam", rorData.city()); + assertEquals("Germany", rorData.country()); + assertEquals(List.of("Facility"), rorData.rorTypes()); + + } + + @ParameterizedTest + @ValueSource(strings = { + "{\"addresses\": [{\"city\": null}], \"country\": {\"country_name\": null}}", + "{\"addresses\": [],\"country\": {}}", + "{}", + }) + void testNullLocationsOrEmptyLocationOrEmptyResponse(String jsonBody) { + RorData rorData = RorScraper.parseData(jsonBody); + + assertNull(rorData.city()); + assertNull(rorData.country()); + } +} From eb1e2c5314fd1d062a2dc597142fd22e8dbdd275 Mon Sep 17 00:00:00 2001 From: "Dusan Mijatovic (PC2026)" Date: Fri, 30 Oct 2026 18:20:49 +0100 Subject: [PATCH 2/4] chore: upgrade postgresql and postgrest to latest minor versions. --- backend-postgrest/Dockerfile | 2 +- database/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend-postgrest/Dockerfile b/backend-postgrest/Dockerfile index 227c29e..45bb174 100644 --- a/backend-postgrest/Dockerfile +++ b/backend-postgrest/Dockerfile @@ -4,4 +4,4 @@ # # SPDX-License-Identifier: Apache-2.0 -FROM postgrest/postgrest:v12.2.8 +FROM postgrest/postgrest:v12.2.12 diff --git a/database/Dockerfile b/database/Dockerfile index e78605e..f2030d1 100644 --- a/database/Dockerfile +++ b/database/Dockerfile @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: Apache-2.0 -FROM postgres:15.12 +FROM postgres:15.19 RUN chmod a+rwx /docker-entrypoint-initdb.d COPY --chown=postgres:postgres *.sh /docker-entrypoint-initdb.d/ COPY --chown=postgres:postgres *.sql /docker-entrypoint-initdb.d/ From 68deef86f3d6ca1ab4697bfb302c672e290cefa1 Mon Sep 17 00:00:00 2001 From: "Dusan Mijatovic (PC2026)" Date: Mon, 2 Nov 2026 16:47:32 +0100 Subject: [PATCH 3/4] fix: upgrade java services --- authentication/Dockerfile | 4 ++-- authentication/pom.xml | 14 +++++++------- background-services/Dockerfile | 4 ++-- background-services/pom.xml | 16 ++++++++-------- scrapers/Dockerfile | 4 ++-- scrapers/pom.xml | 26 +++++++++++++------------- 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/authentication/Dockerfile b/authentication/Dockerfile index 9e3286d..4d32af0 100644 --- a/authentication/Dockerfile +++ b/authentication/Dockerfile @@ -7,7 +7,7 @@ # # SPDX-License-Identifier: Apache-2.0 -FROM maven:3.9.8-eclipse-temurin-21 AS builder +FROM maven:3.10.0-eclipse-temurin-25-noble AS builder WORKDIR /usr/mymaven RUN mkdir src COPY pom.xml . @@ -15,7 +15,7 @@ RUN mvn dependency:go-offline COPY ./src ./src RUN mvn package --offline -FROM eclipse-temurin:21-jre-jammy +FROM eclipse-temurin:25.0.4.1_1-jre-noble WORKDIR /usr/myjava COPY --from=builder /usr/mymaven/target/*-jar-with-dependencies.jar auth.jar CMD java -cp auth.jar nl.esciencecenter.rsd.authentication.Main diff --git a/authentication/pom.xml b/authentication/pom.xml index c4d76ac..fddf87e 100644 --- a/authentication/pom.xml +++ b/authentication/pom.xml @@ -22,8 +22,8 @@ SPDX-License-Identifier: Apache-2.0 UTF-8 - 21 - 21 + 25 + 25 research-software-directory https://sonarcloud.io @@ -41,7 +41,7 @@ SPDX-License-Identifier: Apache-2.0 org.apache.maven.plugins maven-assembly-plugin - 3.7.1 + 3.8.0 @@ -63,7 +63,7 @@ SPDX-License-Identifier: Apache-2.0 org.apache.maven.plugins maven-surefire-plugin - 3.2.5 + 3.5.6 @@ -71,7 +71,7 @@ SPDX-License-Identifier: Apache-2.0 org.jacoco jacoco-maven-plugin - 0.8.12 + 0.8.15 prepare-agent @@ -171,7 +171,7 @@ SPDX-License-Identifier: Apache-2.0 org.apache.maven.surefire surefire-junit-platform - 3.2.5 + 3.5.6 test @@ -206,7 +206,7 @@ SPDX-License-Identifier: Apache-2.0 org.junit junit-bom - 5.10.3 + 6.1.0 pom import diff --git a/background-services/Dockerfile b/background-services/Dockerfile index 1a7ef97..ad54d2f 100644 --- a/background-services/Dockerfile +++ b/background-services/Dockerfile @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: Apache-2.0 -FROM maven:3.9.8-eclipse-temurin-21 AS builder +FROM maven:3.10.0-eclipse-temurin-25-noble AS builder WORKDIR /usr/mymaven RUN mkdir src COPY pom.xml . @@ -11,7 +11,7 @@ RUN mvn dependency:go-offline COPY ./src ./src RUN mvn package --offline -FROM eclipse-temurin:21-jre-jammy +FROM eclipse-temurin:25.0.4.1_1-jre-noble WORKDIR /usr/myjava COPY --from=builder /usr/mymaven/target/*-jar-with-dependencies.jar background-services.jar CMD java -cp background-services.jar nl.esciencecenter.rsd.MainDatabase diff --git a/background-services/pom.xml b/background-services/pom.xml index dd1f845..96c092d 100644 --- a/background-services/pom.xml +++ b/background-services/pom.xml @@ -17,8 +17,8 @@ SPDX-License-Identifier: Apache-2.0 1.0-SNAPSHOT - 21 - 21 + 25 + 25 UTF-8 research-software-directory https://sonarcloud.io @@ -30,14 +30,14 @@ SPDX-License-Identifier: Apache-2.0 org.apache.maven.plugins maven-dependency-plugin - 3.7.1 + 3.8.1 org.apache.maven.plugins maven-assembly-plugin - 3.7.1 + 3.8.0 @@ -57,26 +57,26 @@ SPDX-License-Identifier: Apache-2.0 - + org.postgresql postgresql - 42.7.5 + 42.7.13 org.slf4j slf4j-api - 2.0.16 + 2.0.18 ch.qos.logback logback-classic - 1.5.12 + 1.6.5 diff --git a/scrapers/Dockerfile b/scrapers/Dockerfile index 293db72..1194123 100644 --- a/scrapers/Dockerfile +++ b/scrapers/Dockerfile @@ -5,7 +5,7 @@ # # SPDX-License-Identifier: Apache-2.0 -FROM maven:3.9.8-eclipse-temurin-21 AS builder +FROM maven:3.10.0-eclipse-temurin-25-noble AS builder WORKDIR /usr/mymaven RUN mkdir ./src COPY pom.xml . @@ -13,7 +13,7 @@ RUN mvn dependency:go-offline COPY ./src ./src RUN mvn package --offline -FROM eclipse-temurin:21-jre-jammy +FROM eclipse-temurin:25.0.4.1_1-jre-noble WORKDIR /usr/myjava RUN apt-get update && apt-get --yes install cron nano COPY jobs.cron /etc/cron.d/jobs.cron diff --git a/scrapers/pom.xml b/scrapers/pom.xml index 243f979..fa58713 100644 --- a/scrapers/pom.xml +++ b/scrapers/pom.xml @@ -24,8 +24,8 @@ SPDX-License-Identifier: Apache-2.0 UTF-8 - 21 - 21 + 25 + 25 research-software-directory https://sonarcloud.io @@ -36,14 +36,14 @@ SPDX-License-Identifier: Apache-2.0 org.apache.maven.plugins maven-dependency-plugin - 3.7.1 + 3.8.1 org.apache.maven.plugins maven-assembly-plugin - 3.7.1 + 3.8.0 @@ -65,7 +65,7 @@ SPDX-License-Identifier: Apache-2.0 org.apache.maven.plugins maven-surefire-plugin - 3.2.5 + 3.5.6 @@ -73,7 +73,7 @@ SPDX-License-Identifier: Apache-2.0 org.jacoco jacoco-maven-plugin - 0.8.12 + 0.8.15 prepare-agent @@ -102,7 +102,7 @@ SPDX-License-Identifier: Apache-2.0 org.junit junit-bom - 5.10.3 + 6.1.0 pom import @@ -114,35 +114,35 @@ SPDX-License-Identifier: Apache-2.0 org.slf4j slf4j-api - 2.0.13 + 2.0.18 ch.qos.logback logback-classic - 1.5.6 + 1.6.5 com.auth0 java-jwt - 4.5.0 + 4.6.1 com.google.code.gson gson - 2.11.0 + 2.14.0 org.yaml snakeyaml - 2.4 + 2.6 @@ -164,7 +164,7 @@ SPDX-License-Identifier: Apache-2.0 org.apache.maven.surefire surefire-junit-platform - 3.2.5 + 3.5.6 test From 01556f89a9245af8088c745256bea18764e8b7db Mon Sep 17 00:00:00 2001 From: "Dusan Mijatovic (PC2026)" Date: Mon, 2 Nov 2026 17:47:57 +0100 Subject: [PATCH 4/4] fix: migrate ROR api to v2 --- .../edit/organisations/FindOrganisation.tsx | 2 +- frontend/next-env.d.ts | 4 +- frontend/utils/getROR.test.ts | 28 +- frontend/utils/getROR.ts | 349 +++++++++------- .../rsd/scraper/RsdParsingException.java | 13 + .../nl/esciencecenter/rsd/scraper/Utils.java | 184 ++++++-- .../rsd/scraper/ror/MainRor.java | 55 ++- .../scraper/ror/OrganisationDatabaseData.java | 8 +- .../rsd/scraper/ror/RorData.java | 3 +- .../esciencecenter/rsd/scraper/ror/RorId.java | 17 +- .../scraper/ror/RorPostgrestConnector.java | 17 +- .../rsd/scraper/ror/RorScraper.java | 163 +++----- .../rsd/scraper/ror/RorIdTest.java | 93 +++-- .../ror/RorPostgrestConnectorTest.java | 10 +- .../rsd/scraper/ror/RorScraperTest.java | 394 ++++++++++++++++-- 15 files changed, 930 insertions(+), 410 deletions(-) create mode 100644 scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdParsingException.java diff --git a/frontend/components/software/edit/organisations/FindOrganisation.tsx b/frontend/components/software/edit/organisations/FindOrganisation.tsx index 5c41afb..4f67a53 100644 --- a/frontend/components/software/edit/organisations/FindOrganisation.tsx +++ b/frontend/components/software/edit/organisations/FindOrganisation.tsx @@ -75,8 +75,8 @@ export default function FindOrganisation({onAdd, onCreate}: return (

  • {/* if new option (has input) show label and count */} {`Add "${option.label}"`} diff --git a/frontend/next-env.d.ts b/frontend/next-env.d.ts index 15849f6..03d35cb 100644 --- a/frontend/next-env.d.ts +++ b/frontend/next-env.d.ts @@ -1,7 +1,7 @@ /// /// -import "./.next/types/routes.d.ts"; -import "./.next/types/root-params.d.ts"; +import "./.next/dev/types/routes.d.ts"; +import "./.next/dev/types/root-params.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/pages/api-reference/config/typescript for more information. diff --git a/frontend/utils/getROR.test.ts b/frontend/utils/getROR.test.ts index c9d3bae..959d47e 100644 --- a/frontend/utils/getROR.test.ts +++ b/frontend/utils/getROR.test.ts @@ -25,15 +25,21 @@ beforeEach(() => { it('findInROR calls fetch with search param and json header', async () => { const searchFor = 'ABCD' // mock ROR response - mockFetchJson.mockResolvedValueOnce({items:[{ - id:'test-id', - name:'Test organisation', - country:{country_name:'Test country'}, - addresses:[ - {city: 'Test city'} + mockFetchJson.mockResolvedValueOnce({items: [{ + id: 'test-id', + names: [{value: 'Test organisation', types: ['ror_display']}], + locations: [{geonames_details: {country_name:'Test country', name: 'Test city'}}], + links: [ + { + 'type': 'website', + 'value': 'https://www.test-organisation.com/' + }, + { + 'type': 'wikipedia', + 'value': 'https://en.wikipedia.org/wiki/test-organisation' + } ], - links:[], - types:['Education'] + types: ['Education'] }]}) const resp = await findInROR({searchFor}) @@ -50,11 +56,11 @@ it('findInROR calls fetch with search param and json header', async () => { 'short_description': null, 'description': null, 'ror_id': 'test-id', - 'website': null, + 'website': 'https://www.test-organisation.com/', 'is_tenant': false, 'country': 'Test country', 'city': 'Test city', - 'wikipedia_url': null, + 'wikipedia_url': 'https://en.wikipedia.org/wiki/test-organisation', 'ror_types': ['Education'], 'logo_id': null, 'source': 'ROR', @@ -66,7 +72,7 @@ it('findInROR calls fetch with search param and json header', async () => { expect(mockFetch).toHaveBeenCalledTimes(1) expect(mockFetch).toHaveBeenCalledWith( - `https://api.ror.org/v1/organizations?query=${searchFor}`, + `https://api.ror.org/v2/organizations?query=${searchFor}`, {'headers': {'Content-Type': 'application/json'}} ) }) diff --git a/frontend/utils/getROR.ts b/frontend/utils/getROR.ts index f891efb..f1e7bc0 100644 --- a/frontend/utils/getROR.ts +++ b/frontend/utils/getROR.ts @@ -6,16 +6,17 @@ // // SPDX-License-Identifier: Apache-2.0 -import {AutocompleteOption} from '../types/AutocompleteOptions' -import {OrganisationSource, SearchOrganisation} from '../types/Organisation' +import {AutocompleteOption} from '~/types/AutocompleteOptions' +import {OrganisationSource, SearchOrganisation} from '~/types/Organisation' import {createJsonHeaders} from './fetchHelpers' import {getSlugFromString} from './getSlugFromString' import logger from './logger' export async function findInROR({searchFor}:{searchFor:string}) { try { - // this query will match organisation by name or website values - const url = `https://api.ror.org/v1/organizations?query=${encodeURIComponent(searchFor)}` + // this query will match organisation by name + // https://ror.readme.io/docs/api-query + const url = `https://api.ror.org/v2/organizations?query=${encodeURIComponent(searchFor)}` // make request const resp = await fetch(url, { @@ -24,10 +25,10 @@ export async function findInROR({searchFor}:{searchFor:string}) { ...createJsonHeaders(undefined), } }) + if (resp.status === 200) { const json: any = await resp.json() - const options = buildAutocompleteOptions(json['items']) - return options + return buildAutocompleteOptions(json['items']) } logger(`findInROR FAILED: ${resp.status}: ${resp.statusText}`, 'warn') // we return nothing @@ -39,170 +40,232 @@ export async function findInROR({searchFor}:{searchFor:string}) { } function buildAutocompleteOptions(rorItems: RORItem[]): AutocompleteOption[] { - if (typeof rorItems === 'undefined') return [] - if (rorItems.length === 0) return [] + if (rorItems === undefined || rorItems.length === 0) return [] const options = rorItems.map(item => { - const slug = getSlugFromString(item.name) - return { - // we use ror_id as primary key - key: item?.id ?? slug, - label: item.name, - data: { - id: null, - parent: null, - primary_maintainer: null, - slug, - name: item.name, - short_description: null, - description: null, - ror_id: item.id, - website: item.links[0] ?? null, - is_tenant: false, - country: item?.country?.country_name ?? null, - city: item?.addresses[0]?.city ?? null, - wikipedia_url: item.wikipedia_url ?? null, - ror_types: item.types ?? [], - logo_id: null, - source: 'ROR' as OrganisationSource + try { + const name = item.names + .filter(name => name.types.includes('ror_display')) + .map(name => name.value)[0] + const slug = getSlugFromString(name) + + const wikipediaUrls = item.links.filter(link => link.type === 'wikipedia') + + const websites = item.links ? item.links.filter(link => link.type === 'website') : null + + const firstAddress = item.locations.length ? item.locations[0] : null + + return { + // we use ror_id as primary key + key: item?.id ?? slug, + label: name, + data: { + id: null, + parent: null, + primary_maintainer: null, + slug, + name: name, + short_description: null, + description: null, + ror_id: item.id, + website: websites?.length ? websites[0].value : null, + is_tenant: false, + country: firstAddress?.geonames_details?.country_name ?? null, + city: firstAddress?.geonames_details?.name ?? null, + wikipedia_url: wikipediaUrls.length ? wikipediaUrls[0].value : null, + ror_types: item.types ?? [], + logo_id: null, + source: 'ROR' as OrganisationSource + } } + } catch (e: any) { + logger(`Could not parse ROR v2 item with error ${e} and value ${JSON.stringify(item)}`, 'error') + return null } }) - return options + + return options.filter(item => item !== null) } -// example of ROR item response +// https://github.com/ror-community/ror-schema/blob/master/example_record_v2_1.json // eslint-disable-next-line @typescript-eslint/no-unused-vars -const rorItem = { - 'id': 'https://ror.org/008xxew50', - 'name': 'VU Amsterdam', - 'email_address': null, - 'ip_addresses': [], - 'established': 1880, - 'types': [ - 'Education' - ], - 'relationships': [ - { - 'label': 'Amsterdam UMC Location VUmc', - 'type': 'Related', - 'id': 'https://ror.org/00q6h8f30' - }, +const rorV2Item = { + 'locations': [ { - 'label': 'Spinoza Centre for Neuroimaging', - 'type': 'Related', - 'id': 'https://ror.org/05kgbsy64' - }, + 'geonames_id': 5378538, + 'geonames_details': { + 'continent_code': 'NA', + 'continent_name': 'North America', + 'country_code': 'US', + 'country_name': 'United States', + 'country_subdivision_code': 'CA', + 'country_subdivision_name': 'California', + 'lat': 37.802168, + 'lng': -122.271281, + 'name': 'Oakland' + } + } + ], + 'established': 1868, + 'external_ids': [ { - 'label': 'Advanced Research Center for Nanolithography (Netherlands)', - 'type': 'Child', - 'id': 'https://ror.org/04xe7ws48' + 'type': 'fundref', + 'all': [ + '100005595', + '100009350', + '100004802', + '100010574', + '100005188', + '100005192' + ], + 'preferred': '100005595' }, { - 'label': 'Amsterdam Neuroscience', - 'type': 'Child', - 'id': 'https://ror.org/01x2d9f70' + 'type': 'grid', + 'all': [ + 'grid.30389.31' + ], + 'preferred': 'grid.30389.31' }, { - 'label': 'EMGO Institute for Health and Care Research', - 'type': 'Child', - 'id': 'https://ror.org/0258apj61' + 'type': 'isni', + 'all': [ + '0000 0001 2348 0690' + ], + 'preferred': null } ], - 'addresses': [ - { - 'lat': 52.333941, - 'lng': 4.865709, - 'state': 'Noord-Holland', - 'state_code': 'NL-NH', - 'city': 'Amsterdam', - 'geonames_city': { - 'id': 2759794, - 'city': 'Amsterdam', - 'geonames_admin1': { - 'name': 'North Holland', - 'id': 2749879, - 'ascii_name': 'North Holland', - 'code': 'NL.07' - }, - 'geonames_admin2': { - 'name': 'Gemeente Amsterdam', - 'id': 2759793, - 'ascii_name': 'Gemeente Amsterdam', - 'code': 'NL.07.0363' - }, - 'license': { - 'attribution': 'Data from geonames.org under a CC-BY 3.0 license', - 'license': 'http://creativecommons.org/licenses/by/3.0/' - }, - 'nuts_level1': { - 'name': 'WEST-NEDERLAND', - 'code': 'NL3' - }, - 'nuts_level2': { - 'name': 'Noord-Holland', - 'code': 'NL32' - }, - 'nuts_level3': { - 'name': 'Groot-Amsterdam', - 'code': 'NL326' - } - }, - 'postcode': null, - 'primary': false, - 'line': null, - 'country_geonames_id': 2750405 - } + 'id': 'https://ror.org/00pjdza24', + 'domains': [ + 'universityofcalifornia.edu' ], 'links': [ - 'http://www.vu.nl/en/' - ], - 'aliases': [], - 'acronyms': [ - 'VU' + { + 'type': 'website', + 'value': 'http://www.universityofcalifornia.edu/' + }, + { + 'type': 'wikipedia', + 'value': 'http://en.wikipedia.org/wiki/University_of_California' + } ], - 'status': 'active', - 'wikipedia_url': 'http://en.wikipedia.org/wiki/VU_University_Amsterdam', - 'labels': [ + 'names': [ + { + 'value': 'UC', + 'types': [ + 'acronym' + ], + 'lang': 'en' + }, + { + 'value': 'UC System', + 'types': [ + 'alias' + ], + 'lang': 'en' + }, + { + 'value': 'University of California System', + 'types': [ + 'ror_display', + 'label' + ], + 'lang': 'en' + }, { - 'label': 'Vrije Universiteit Amsterdam', - 'iso639': 'nl' + 'value': 'Université de Californie', + 'types': [ + 'label' + ], + 'lang': 'fr' } ], - 'country': { - 'country_name': 'Netherlands', - 'country_code': 'NL' - }, - 'external_ids': { - 'ISNI': { - 'preferred': null, - 'all': [ - '0000 0004 1754 9227' - ] + 'relationships': [ + { + 'id': 'https://ror.org/02jbv0t02', + 'label': 'Lawrence Berkeley National Laboratory', + 'type': 'related' }, - 'FundRef': { - 'preferred': null, - 'all': [ - '501100001833' - ] + { + 'id': 'https://ror.org/03yrm5c26', + 'label': 'California Digital Library', + 'type': 'child' }, - 'OrgRef': { - 'preferred': null, - 'all': [ - '324167' - ] + { + 'id': 'https://ror.org/00zv0wd17', + 'label': 'Center for Information Technology Research in the Interest of Society', + 'type': 'child' }, - 'Wikidata': { - 'preferred': null, - 'all': [ - 'Q1065414' - ] + { + 'id': 'https://ror.org/03t0t6y08', + 'label': 'University of California Division of Agriculture and Natural Resources', + 'type': 'child' + }, + { + 'id': 'https://ror.org/01an7q238', + 'label': 'University of California, Berkeley', + 'type': 'child' + }, + { + 'id': 'https://ror.org/05rrcem69', + 'label': 'University of California, Davis', + 'type': 'child' + }, + { + 'id': 'https://ror.org/04gyf1771', + 'label': 'University of California, Irvine', + 'type': 'child' + }, + { + 'id': 'https://ror.org/046rm7j60', + 'label': 'University of California, Los Angeles', + 'type': 'child' + }, + { + 'id': 'https://ror.org/00d9ah105', + 'label': 'University of California, Merced', + 'type': 'child' + }, + { + 'id': 'https://ror.org/03nawhv43', + 'label': 'University of California, Riverside', + 'type': 'child' + }, + { + 'id': 'https://ror.org/0168r3w48', + 'label': 'University of California, San Diego', + 'type': 'child' + }, + { + 'id': 'https://ror.org/043mz5j54', + 'label': 'University of California, San Francisco', + 'type': 'child' + }, + { + 'id': 'https://ror.org/02t274463', + 'label': 'University of California, Santa Barbara', + 'type': 'child' + }, + { + 'id': 'https://ror.org/03s65by71', + 'label': 'University of California, Santa Cruz', + 'type': 'child' + } + ], + 'status': 'active', + 'types': [ + 'education' + ], + 'admin': { + 'created': { + 'date': '2020-04-25', + 'schema_version': '1.0' }, - 'GRID': { - 'preferred': 'grid.12380.38', - 'all': 'grid.12380.38' + 'last_modified': { + 'date': '2022-10-18', + 'schema_version': '2.0' } } } -export type RORItem = typeof rorItem +type RORItem = typeof rorV2Item diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdParsingException.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdParsingException.java new file mode 100644 index 0000000..390e43f --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdParsingException.java @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper; + +public class RsdParsingException extends Exception { + + public RsdParsingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Utils.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Utils.java index 23c949c..3a94974 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Utils.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Utils.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2026 Netherlands eScience Center // SPDX-FileCopyrightText: 2022 Christian Meeßen (GFZ) // SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) // SPDX-FileCopyrightText: 2022 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences @@ -13,13 +13,11 @@ import com.auth0.jwt.algorithms.Algorithm; import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; import java.net.URI; +import java.net.URISyntaxException; import java.net.URLEncoder; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -30,8 +28,10 @@ import java.time.format.DateTimeFormatter; import java.util.Base64; import java.util.Date; +import java.util.Locale; import java.util.UUID; -import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class Utils { @@ -40,7 +40,55 @@ public class Utils { // Default timeout used for http connections. private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(30); - private Utils() { + private Utils() {} + + /** + * removes all leading, trailing and double white space from a string + * + * @param s the string to sanitise + * @return the sanitised string or null if the input was null + */ + public static String sanitiseWhitespace(String s) { + if (s == null) { + return null; + } + + return s.replaceAll("\\s+", " ").strip(); + } + + /** + * + * @param s the string to slugify, cannot be null + * @return the input for which the letters A-Z are converted to lower case, + * the digits 0-9 are unchanged, + * other characters have been converted to a dash, + * leading, trailing and more than one duplicate dashes have been removed + */ + public static String slugify(String s) { + return s.toLowerCase(Locale.ENGLISH).replaceAll("[^a-z0-9]+", "-").replaceAll("^-", "").replaceAll("-$", ""); + } + + /** + * + * @param s the candidate URL to check + * @return if the input is a valid HTTP(S) URL + */ + public static boolean isUrl(String s) { + if (s == null) { + return false; + } + + if (!s.startsWith("https://") && !s.startsWith("http://")) { + return false; + } + + try { + new URI(s); + } catch (URISyntaxException _) { + return false; + } + + return true; } /** @@ -73,11 +121,17 @@ public static String urlEncode(String value) { * @throws InterruptedException * @throws RsdResponseException */ - public static String get(String uri, String... headers) throws IOException, InterruptedException, RsdResponseException { + public static String get(String uri, String... headers) + throws IOException, InterruptedException, RsdResponseException { HttpResponse response = getAsHttpResponse(uri, headers); if (response.statusCode() >= 300) { - throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response"); + throw new RsdResponseException( + response.statusCode(), + response.uri(), + response.body(), + "Unexpected response" + ); } return response.body(); @@ -86,17 +140,44 @@ public static String get(String uri, String... headers) throws IOException, Inte /** * Performs a GET request with given headers and returns the entire http response. * - * @param uri The encoded URI + * @param uri The URI * @param headers (Optional) Variable amount of headers. Number of arguments must be a multiple of two. * @return The response as a String. - * @throws IOException - * @throws InterruptedException + * @throws IOException when the underlying HTTP request throws it + * @throws InterruptedException when this operation is interrupted */ - public static HttpResponse getAsHttpResponse(String uri, String... headers) throws IOException, InterruptedException { - HttpRequest.Builder httpRequestBuilder = HttpRequest.newBuilder() - .GET() - .timeout(DEFAULT_TIMEOUT) - .uri(URI.create(uri)); + public static HttpResponse getAsHttpResponse(URI uri, String... headers) + throws IOException, InterruptedException { + return getAsHttpResponse(uri, DEFAULT_TIMEOUT, headers); + } + + /** + * Performs a GET request with given headers and returns the entire http response. + * + * @param uri The encoded URI + * @param headers (Optional) Variable amount of headers. Number of arguments must be a multiple of two. + * @return The response as a String. + * @throws IOException when the underlying HTTP request throws it + * @throws InterruptedException when this operation is interrupted + */ + public static HttpResponse getAsHttpResponse(String uri, String... headers) + throws IOException, InterruptedException { + return getAsHttpResponse(URI.create(uri), headers); + } + + /** + * Performs a GET request with given headers and timeout and returns the entire http response. + * + * @param uri The encoded URI + * @param timeout the timeout of the HTTP request + * @param headers (Optional) Variable amount of headers. Number of arguments must be a multiple of two. + * @return The response as a String. + * @throws IOException when the underlying HTTP request throws it + * @throws InterruptedException when this operation is interrupted + */ + public static HttpResponse getAsHttpResponse(URI uri, Duration timeout, String... headers) + throws IOException, InterruptedException { + HttpRequest.Builder httpRequestBuilder = HttpRequest.newBuilder().GET().timeout(timeout).uri(uri); if (headers != null && headers.length > 0 && headers.length % 2 == 0) { httpRequestBuilder.headers(headers); } @@ -136,7 +217,9 @@ public static String getAsAdmin(String uri) { } if (response.statusCode() >= 300) { - throw new RuntimeException("Error fetching data from endpoint " + uri + " with response: " + response.body()); + throw new RuntimeException( + "Error fetching data from endpoint " + uri + " with response: " + response.body() + ); } return response.body(); } @@ -171,7 +254,9 @@ public static String post(String uri, String body, String... extraHeaders) { } if (response.statusCode() >= 300) { - throw new RuntimeException("Error fetching data from endpoint " + uri + " with response: " + response.body()); + throw new RuntimeException( + "Error fetching data from endpoint " + uri + " with response: " + response.body() + ); } return response.body(); } @@ -208,7 +293,9 @@ public static String postAsAdmin(String uri, String json, String... extraHeaders } if (response.statusCode() >= 300) { - throw new RuntimeException("Error fetching data from endpoint " + uri + " with response: " + response.body()); + throw new RuntimeException( + "Error fetching data from endpoint " + uri + " with response: " + response.body() + ); } return response.body(); } @@ -232,12 +319,21 @@ private static JsonObject basicData(String serviceName, String tableName, UUID r } public static void saveExceptionInDatabase(String serviceName, String tableName, UUID referenceId, Exception e) { + if (e instanceof RsdResponseException re) { + saveExceptionInDatabase(serviceName, tableName, referenceId, re); + return; + } JsonObject logData = basicData(serviceName, tableName, referenceId, e); postAsAdmin(Config.backendBaseUrl() + "/backend_log", logData.toString()); } - public static void saveExceptionInDatabase(String serviceName, String tableName, UUID referenceId, RsdResponseException e) { + public static void saveExceptionInDatabase( + String serviceName, + String tableName, + UUID referenceId, + RsdResponseException e + ) { JsonObject logData = basicData(serviceName, tableName, referenceId, e); JsonObject other = new JsonObject(); @@ -250,7 +346,12 @@ public static void saveExceptionInDatabase(String serviceName, String tableName, postAsAdmin(Config.backendBaseUrl() + "/backend_log", logData.toString()); } - public static void saveExceptionInDatabase(String serviceName, String tableName, UUID referenceId, RsdRateLimitException e) { + public static void saveExceptionInDatabase( + String serviceName, + String tableName, + UUID referenceId, + RsdRateLimitException e + ) { JsonObject logData = basicData(serviceName, tableName, referenceId, e); JsonObject other = new JsonObject(); @@ -263,7 +364,15 @@ public static void saveExceptionInDatabase(String serviceName, String tableName, postAsAdmin(Config.backendBaseUrl() + "/backend_log", logData.toString()); } - public static void saveErrorMessageInDatabase(String message, String tableName, String columnName, String primaryKey, String primaryKeyName, ZonedDateTime scrapedAt, String scrapedAtName) { + public static void saveErrorMessageInDatabase( + String message, + String tableName, + String columnName, + String primaryKey, + String primaryKeyName, + ZonedDateTime scrapedAt, + String scrapedAtName + ) { JsonObject body = new JsonObject(); if (columnName != null) { body.addProperty(columnName, message); @@ -304,12 +413,15 @@ public static String patchAsAdmin(String uri, String json, String... extraHeader throw new RuntimeException(e); } if (response.statusCode() >= 300) { - throw new RuntimeException("Error fetching data from endpoint " + uri + " with response: " + response.body()); + throw new RuntimeException( + "Error fetching data from endpoint " + uri + " with response: " + response.body() + ); } return response.body(); } - public static String deleteAsAdmin(String uri, String... extraHeaders) throws IOException, InterruptedException, RsdResponseException { + public static String deleteAsAdmin(String uri, String... extraHeaders) + throws IOException, InterruptedException, RsdResponseException { String jwtString = adminJwt(); HttpRequest.Builder builder = HttpRequest.newBuilder() .method("DELETE", HttpRequest.BodyPublishers.noBody()) @@ -325,7 +437,12 @@ public static String deleteAsAdmin(String uri, String... extraHeaders) throws IO response = client.send(request, HttpResponse.BodyHandlers.ofString()); } if (response.statusCode() >= 300) { - throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Error deleting data as admin"); + throw new RsdResponseException( + response.statusCode(), + response.uri(), + response.body(), + "Error deleting data as admin" + ); } return response.body(); } @@ -343,15 +460,6 @@ public static String stringOrNull(JsonElement e) { return e == null || !e.isJsonPrimitive() ? null : e.getAsString(); } - public static T safelyGetOrNull(JsonElement element, Function extractor) { - try { - return extractor.apply(element); - } catch (RuntimeException e) { - LOGGER.warn("Exception extracting data from JSON: " + element, e); - return null; - } - } - public static Integer integerOrNull(JsonElement e) { return e == null || !e.isJsonPrimitive() ? null : e.getAsInt(); } @@ -363,9 +471,9 @@ public static Integer integerOrNull(JsonElement e) { * @return the filter. */ public static String atLeastOneHourAgoFilter(String scrapedAtColumnName) { - String oneHourAgoEncoded = urlEncode(ZonedDateTime.now() - .minusHours(1) - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + String oneHourAgoEncoded = urlEncode( + ZonedDateTime.now().minusHours(1).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + ); return "or=(%s.is.null,%s.lte.%s)".formatted(scrapedAtColumnName, scrapedAtColumnName, oneHourAgoEncoded); } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/MainRor.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/MainRor.java index 7a6d528..966fe37 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/MainRor.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/MainRor.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2023 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center // SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) // SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences // @@ -7,33 +7,36 @@ package nl.esciencecenter.rsd.scraper.ror; +import java.io.IOException; +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.concurrent.CompletableFuture; import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdParsingException; import nl.esciencecenter.rsd.scraper.RsdResponseException; import nl.esciencecenter.rsd.scraper.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.time.ZonedDateTime; -import java.util.Collection; -import java.util.concurrent.CompletableFuture; - public class MainRor { private static final Logger LOGGER = LoggerFactory.getLogger(MainRor.class); private static final int SCRAPING_LIMIT = Config.maxRequestsRor(); public static void main(String[] args) { - LOGGER.info("Start scraping ROR data."); + LOGGER.info("Start harvesting ROR data."); long t1 = System.currentTimeMillis(); scrapeLocationData(); long time = System.currentTimeMillis() - t1; - LOGGER.info("Done scraping ROR data ({} ms).", time); + LOGGER.info("Done harvesting ROR data ({} ms).", time); } private static void scrapeLocationData() { RorPostgrestConnector organisationsInRSD = new RorPostgrestConnector(); - Collection organisationsToScrape = organisationsInRSD.organisationsWithoutLocation(SCRAPING_LIMIT); + Collection organisationsToScrape = + organisationsInRSD.leastRecentlyScrapedOrganisations(SCRAPING_LIMIT); + LOGGER.info("Harvesting {} organisations from ROR", organisationsToScrape.size()); + CompletableFuture[] futures = new CompletableFuture[organisationsToScrape.size()]; ZonedDateTime scrapedAt = ZonedDateTime.now(); int i = 0; @@ -46,17 +49,38 @@ private static void scrapeLocationData() { try { RorScraper rorScraper = new RorScraper(organisation.rorId()); RorData data = rorScraper.scrapeData(); - OrganisationDatabaseData updatedOrganisationDatabaseData = new OrganisationDatabaseData(organisation.id(), organisation.rorId(), scrapedAt, data); + OrganisationDatabaseData updatedOrganisationDatabaseData = new OrganisationDatabaseData( + organisation.id(), + organisation.rorId(), + scrapedAt, + data + ); organisationsInRSD.saveLocationData(updatedOrganisationDatabaseData); - } catch (RsdResponseException | IOException | InterruptedException e) { + } catch (RsdResponseException | RsdParsingException | IOException | InterruptedException e) { Utils.saveExceptionInDatabase("ROR location scraper", tableName, organisation.id(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), tableName, columnName, organisation.id().toString(), primaryKeyName, scrapedAt, scrapedAtName); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + tableName, + columnName, + organisation.id().toString(), + primaryKeyName, + scrapedAt, + scrapedAtName + ); if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } - } catch (Exception e) { + } catch (RuntimeException e) { Utils.saveExceptionInDatabase("ROR location scraper", tableName, organisation.id(), e); - Utils.saveErrorMessageInDatabase("Unknown error", tableName, columnName, organisation.id().toString(), primaryKeyName, scrapedAt, scrapedAtName); + Utils.saveErrorMessageInDatabase( + "Unknown error", + tableName, + columnName, + organisation.id().toString(), + primaryKeyName, + scrapedAt, + scrapedAtName + ); } }); futures[i] = future; @@ -64,5 +88,4 @@ private static void scrapeLocationData() { } CompletableFuture.allOf(futures).join(); } - } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/OrganisationDatabaseData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/OrganisationDatabaseData.java index 9684f77..faf082f 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/OrganisationDatabaseData.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/OrganisationDatabaseData.java @@ -10,10 +10,4 @@ import java.time.ZonedDateTime; import java.util.UUID; -public record OrganisationDatabaseData( - UUID id, - RorId rorId, - ZonedDateTime rorScrapedAt, - RorData data -) { -} +public record OrganisationDatabaseData(UUID id, RorId rorId, ZonedDateTime rorScrapedAt, RorData data) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorData.java index 4845f86..bd2cd13 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorData.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorData.java @@ -15,5 +15,4 @@ public record RorData( List rorNames, Double lat, Double lon -) { -} +) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorId.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorId.java index 4a098ce..2324164 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorId.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorId.java @@ -21,8 +21,11 @@ public class RorId { private static final String ROR_BASE_URL = "https://ror.org/"; // https://ror.org/blog/2024-04-15-announcing-ror-v2/ - private static final String ROR_BASE_API_V1_URL = "https://api.ror.org/v1/organizations/"; - private static final Pattern ROR_URL_PATTERN = Pattern.compile("^https://ror\\.org/(0[a-hj-km-np-tv-z|\\d]{6}\\d{2})$"); + // https://ror.readme.io/changelog/2025-07-01-sunset-of-version-1 + private static final String ROR_BASE_API_V2_URL = "https://api.ror.org/v2/organizations/"; + private static final Pattern ROR_URL_PATTERN = Pattern.compile( + "^https://ror\\.org/(0[a-hj-km-np-tv-z|\\d]{6}\\d{2})$" + ); private final String id; @@ -56,11 +59,13 @@ public static boolean isValidRorUrl(String url) { public static RorId fromUrlString(String url) { Objects.requireNonNull(url); if (!isValidRorUrl(url)) { - throw new IllegalArgumentException("The url %s is not a valid ROR ID".formatted(url)); + throw new IllegalArgumentException("The URL %s is not a valid ROR ID".formatted(url)); } Matcher matcher = ROR_URL_PATTERN.matcher(url); - matcher.find(); + if (!matcher.find()) { + throw new AssertionError("URL %s passed isValidRorUrl but now no match found".formatted(url)); + } String id = matcher.group(1); return new RorId(id); } @@ -69,8 +74,8 @@ public URI asUrl() { return URI.create(ROR_BASE_URL + id); } - public URI asApiV1Url() { - return URI.create(ROR_BASE_API_V1_URL + id); + public URI asApiV2Url() { + return URI.create(ROR_BASE_API_V2_URL + id); } @Override diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnector.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnector.java index 6a992e4..c8718c7 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnector.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnector.java @@ -12,24 +12,25 @@ import com.google.gson.JsonNull; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.Config; -import nl.esciencecenter.rsd.scraper.Utils; - import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.UUID; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.Utils; public class RorPostgrestConnector { + private final String backendUrl; public RorPostgrestConnector() { this.backendUrl = Config.backendBaseUrl(); } - public Collection organisationsWithoutLocation(int limit) { - String filter = "organisation?ror_id=not.is.null&order=ror_scraped_at.asc.nullsfirst&select=id,ror_id&limit=" + limit; + public Collection leastRecentlyScrapedOrganisations(int limit) { + String filter = + "organisation?ror_id=not.is.null&order=ror_scraped_at.asc.nullsfirst&select=id,ror_id&limit=" + limit; String data = Utils.getAsAdmin(backendUrl + "/" + filter); return parseBasicJsonData(data); } @@ -73,8 +74,10 @@ public void saveLocationData(OrganisationDatabaseData organisationData) { jsonObject.addProperty("lat", organisationData.data().lat()); jsonObject.addProperty("lon", organisationData.data().lon()); - jsonObject.addProperty("ror_scraped_at", organisationData.rorScrapedAt() - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + jsonObject.addProperty( + "ror_scraped_at", + organisationData.rorScrapedAt().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + ); jsonObject.add("ror_last_error", JsonNull.INSTANCE); Utils.patchAsAdmin(backendUrl + "/organisation?id=eq." + organisationData.id(), jsonObject.toString()); diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorScraper.java index 61c2e37..66291e3 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/ror/RorScraper.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2023 - 2025 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2023 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2026 Netherlands eScience Center // SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) // SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences // @@ -7,16 +7,17 @@ package nl.esciencecenter.rsd.scraper.ror; +import com.google.gson.JsonArray; import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.RsdResponseException; -import nl.esciencecenter.rsd.scraper.Utils; - import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Objects; +import nl.esciencecenter.rsd.scraper.RsdParsingException; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; public class RorScraper { @@ -27,107 +28,65 @@ public RorScraper(RorId rorId) { } private String getFromApi() throws IOException, InterruptedException, RsdResponseException { - // e.g https://api.ror.org/organizations/04tsk2644 - return Utils.get(rorId.asApiV1Url().toString()); + // e.g https://api.ror.org/v2/organizations/04tsk2644 + return Utils.get(rorId.asApiV2Url().toString()); } - public RorData scrapeData() throws RsdResponseException, IOException, InterruptedException { + public RorData scrapeData() throws RsdResponseException, IOException, InterruptedException, RsdParsingException { String json = getFromApi(); - return parseData(json); + return parseV2Data(json); } - static RorData parseData(String json) { - JsonElement jsonElement = JsonParser.parseString(json); - final String addressesKey = "addresses"; - - String country = Utils.safelyGetOrNull(jsonElement, j -> j - .getAsJsonObject() - .getAsJsonObject("country") - .getAsJsonPrimitive("country_name") - .getAsString()); - String city = Utils.safelyGetOrNull(jsonElement, j -> j - .getAsJsonObject() - .getAsJsonArray(addressesKey) - .get(0) - .getAsJsonObject() - .getAsJsonPrimitive("city") - .getAsString()); - String wikipediaUrl = Utils.safelyGetOrNull(jsonElement, j -> { - JsonElement wikiElement = j - .getAsJsonObject() - .get("wikipedia_url"); - return wikiElement.isJsonPrimitive() ? wikiElement.getAsString() : null; - }); - if (wikipediaUrl != null && wikipediaUrl.isBlank()) { - wikipediaUrl = null; - } - List rorTypes = Utils.safelyGetOrNull(jsonElement, j -> j - .getAsJsonObject() - .getAsJsonArray("types") - .asList() - .stream() - .map(JsonElement::getAsString) - .toList() - ); - - List rorNames = new ArrayList<>(); - List rorAliases = Utils.safelyGetOrNull(jsonElement, j -> j - .getAsJsonObject() - .getAsJsonArray("aliases") - .asList() - .stream() - .map(alias -> alias.getAsJsonPrimitive().getAsString()) - .toList() - ); - if (rorAliases != null) { - rorNames.addAll(rorAliases); - } - List rorAcronyms = Utils.safelyGetOrNull(jsonElement, j -> j - .getAsJsonObject() - .getAsJsonArray("acronyms") - .asList() - .stream() - .map(acronym -> acronym.getAsJsonPrimitive().getAsString()) - .toList() - ); - if (rorAcronyms != null) { - rorNames.addAll(rorAcronyms); - } - List rorLabels = Utils.safelyGetOrNull(jsonElement, j -> j - .getAsJsonObject() - .getAsJsonArray("labels") - .asList() - .stream() - .map(label -> label.getAsJsonObject().getAsJsonPrimitive("label").getAsString()) - .toList() - ); - if (rorLabels != null) { - rorNames.addAll(rorLabels); - } + // the full schema is at https://github.com/ror-community/ror-schema/blob/master/ror_schema_v2_1.json + static RorData parseV2Data(String json) throws RsdParsingException { + try { + JsonObject parsedJson = JsonParser.parseString(json).getAsJsonObject(); - Double lat = Utils.safelyGetOrNull(jsonElement, j -> j - .getAsJsonObject() - .getAsJsonArray(addressesKey) - .get(0) - .getAsJsonObject() - .getAsJsonPrimitive("lat") - .getAsDouble()); - Double lon = Utils.safelyGetOrNull(jsonElement, j -> j - .getAsJsonObject() - .getAsJsonArray(addressesKey) - .get(0) - .getAsJsonObject() - .getAsJsonPrimitive("lng") - .getAsDouble()); - - return new RorData( - country, - city, - wikipediaUrl, - rorTypes == null ? Collections.emptyList() : rorTypes, - rorNames, - lat, - lon - ); + JsonArray addressesJson = parsedJson.getAsJsonArray("locations"); + // what to do when multiple addresses exist? + JsonObject firstAddressJson = addressesJson.get(0).getAsJsonObject().getAsJsonObject("geonames_details"); + + JsonElement countryJson = firstAddressJson.get("country_name"); + String country = + countryJson != null && countryJson.isJsonPrimitive() + ? countryJson.getAsJsonPrimitive().getAsString() + : null; + + String city = firstAddressJson.getAsJsonPrimitive("name").getAsString(); + + JsonElement latJson = firstAddressJson.get("lat"); + Double lat = + latJson != null && latJson.isJsonPrimitive() ? latJson.getAsJsonPrimitive().getAsDouble() : null; + + JsonElement lonJson = firstAddressJson.get("lng"); + Double lon = + lonJson != null && lonJson.isJsonPrimitive() ? lonJson.getAsJsonPrimitive().getAsDouble() : null; + + JsonArray linksJson = parsedJson.getAsJsonArray("links"); + String wikipediaUrl = null; + for (JsonElement jsonElement : linksJson) { + JsonObject linkObjectJson = jsonElement.getAsJsonObject(); + if (linkObjectJson.getAsJsonPrimitive("type").getAsString().equals("wikipedia")) { + wikipediaUrl = linkObjectJson.getAsJsonPrimitive("value").getAsString(); + break; + } + } + + JsonArray typesJson = parsedJson.getAsJsonArray("types"); + List rorTypes = new ArrayList<>(typesJson.size()); + for (JsonElement jsonElement : typesJson) { + rorTypes.add(jsonElement.getAsString()); + } + + JsonArray namesJson = parsedJson.getAsJsonArray("names"); + List rorNames = new ArrayList<>(namesJson.size()); + for (JsonElement jsonElement : namesJson) { + rorNames.add(jsonElement.getAsJsonObject().getAsJsonPrimitive("value").getAsString()); + } + + return new RorData(country, city, wikipediaUrl, rorTypes, rorNames, lat, lon); + } catch (RuntimeException e) { + throw new RsdParsingException("Error parsing JSON data from ROR API", e); + } } } diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorIdTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorIdTest.java index 8d029c7..ae02b06 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorIdTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorIdTest.java @@ -5,71 +5,78 @@ package nl.esciencecenter.rsd.scraper.ror; +import java.net.URI; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import java.net.URI; - class RorIdTest { @ParameterizedTest - @ValueSource(strings = { - "https://ror.org/14tsk2644", // ID should start with 0 - "https://ror.org/0ltsk2644", // character 'l' not allowed - "https://ror.org/04tsk26444", // ID longer than 9 characters - "https://ror.org/04tsk264", // // ID shorter than 9 characters - "ror.org/04tsk2644", // should start with https:// - "", - }) + @ValueSource( + strings = { + "https://ror.org/14tsk2644", // ID should start with 0 + "https://ror.org/0ltsk2644", // character 'l' not allowed + "https://ror.org/04tsk26444", // ID longer than 9 characters + "https://ror.org/04tsk264", // // ID shorter than 9 characters + "ror.org/04tsk2644", // should start with https:// + "", + } + ) void givenInvalidRorIds_whenTesting_thenFalseReturned(String url) { Assertions.assertFalse(RorId.isValidRorUrl(url)); } @ParameterizedTest - @ValueSource(strings = { - "https://ror.org/14tsk2644", // ID should start with 0 - "https://ror.org/0ltsk2644", // character 'l' not allowed - "https://ror.org/04tsk26444", // ID longer than 9 characters - "https://ror.org/04tsk264", // // ID shorter than 9 characters - "ror.org/04tsk2644", // should start with https:// - "", - }) + @ValueSource( + strings = { + "https://ror.org/14tsk2644", // ID should start with 0 + "https://ror.org/0ltsk2644", // character 'l' not allowed + "https://ror.org/04tsk26444", // ID longer than 9 characters + "https://ror.org/04tsk264", // // ID shorter than 9 characters + "ror.org/04tsk2644", // should start with https:// + "", + } + ) void givenInvalidRorIds_whenCreatingInstance_thenExceptionThrown(String url) { Assertions.assertThrows(IllegalArgumentException.class, () -> RorId.fromUrlString(url)); } @ParameterizedTest - @ValueSource(strings = { - "https://ror.org/04tsk2644", - "https://ror.org/05qghxh33", - "https://ror.org/01e6qks80", - "https://ror.org/00wjc7c48", - "https://ror.org/01swzsf04", - "https://ror.org/02jbv0t02", - "https://ror.org/05grdyy37", - "https://ror.org/05f950310", - "https://ror.org/04ke6ht85", - "https://ror.org/018dfmf50", - }) + @ValueSource( + strings = { + "https://ror.org/04tsk2644", + "https://ror.org/05qghxh33", + "https://ror.org/01e6qks80", + "https://ror.org/00wjc7c48", + "https://ror.org/01swzsf04", + "https://ror.org/02jbv0t02", + "https://ror.org/05grdyy37", + "https://ror.org/05f950310", + "https://ror.org/04ke6ht85", + "https://ror.org/018dfmf50", + } + ) void givenValidRorIds_whenTesting_thenTrueReturned(String url) { Assertions.assertTrue(RorId.isValidRorUrl(url)); } @ParameterizedTest - @ValueSource(strings = { - "https://ror.org/04tsk2644", - "https://ror.org/05qghxh33", - "https://ror.org/01e6qks80", - "https://ror.org/00wjc7c48", - "https://ror.org/01swzsf04", - "https://ror.org/02jbv0t02", - "https://ror.org/05grdyy37", - "https://ror.org/05f950310", - "https://ror.org/04ke6ht85", - "https://ror.org/018dfmf50", - }) + @ValueSource( + strings = { + "https://ror.org/04tsk2644", + "https://ror.org/05qghxh33", + "https://ror.org/01e6qks80", + "https://ror.org/00wjc7c48", + "https://ror.org/01swzsf04", + "https://ror.org/02jbv0t02", + "https://ror.org/05grdyy37", + "https://ror.org/05f950310", + "https://ror.org/04ke6ht85", + "https://ror.org/018dfmf50", + } + ) void givenValidRorIds_whenCreatingInstance_thenNoExceptionThrown(String url) { RorId rorId = Assertions.assertDoesNotThrow(() -> RorId.fromUrlString(url)); Assertions.assertNotNull(rorId); @@ -80,6 +87,6 @@ void givenValidRorId_whenGettingUrls_thenCorrectUrlsReturned() { RorId rorId = RorId.fromUrlString("https://ror.org/04tsk2644"); Assertions.assertEquals(URI.create("https://ror.org/04tsk2644"), rorId.asUrl()); - Assertions.assertEquals(URI.create("https://api.ror.org/v1/organizations/04tsk2644"), rorId.asApiV1Url()); + Assertions.assertEquals(URI.create("https://api.ror.org/v2/organizations/04tsk2644"), rorId.asApiV2Url()); } } diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnectorTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnectorTest.java index 83e2971..7973ed4 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnectorTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorPostgrestConnectorTest.java @@ -7,19 +7,19 @@ package nl.esciencecenter.rsd.scraper.ror; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import java.util.Collection; import java.util.UUID; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; +import org.junit.jupiter.api.Test; class RorPostgrestConnectorTest { @Test void testParseBasicJsonData() { - String data = "[{\"id\":\"52b13a36-334b-429a-9cbb-5215264b36d2\",\"parent\":null,\"primary_maintainer\":\"22f87e87-7fd9-4716-9dc6-e790372cac4c\",\"slug\":\"gfz\",\"name\":\"Helmholtz Centre Potsdam GFZ German Research Centre for Geosciences\",\"ror_id\":\"https://ror.org/04z8jg394\",\"website\":\"https://www.gfz-potsdam.de\",\"is_tenant\":true,\"created_at\":\"2022-07-26T19:34:55.173342+00:00\",\"updated_at\":\"2023-05-17T06:11:51.338233+00:00\",\"description\":\"# About GFZ Potsdam\n\nThe GFZ is Germany’s national research center for the solid Earth Sciences. Our mission is to deepen the knowledge of the dynamics of the solid Earth, and to develop solutions for grand challenges facing society. These challenges include anticipating the hazards arising from the Earth’s dynamic systems and mitigating the associated risks to society; securing our habitat under the pressure of global change; and supplying energy and mineral resources for a rapidly growing population in a sustainable manner and without harming the environment. \n\n## Research Software at GFZ\n\nTo learn more about the research software policy at GFZ, visit [gfz-potsdam.de/en/software](https://www.gfz-potsdam.de/en/software).\n\n\",\"logo_id\":\"da40b40113fdfb1d3ce40f58ecb784fe33ba40bb\",\"country\":null,\"city\":null,\"short_description\":null}]"; + String data = + "[{\"id\":\"52b13a36-334b-429a-9cbb-5215264b36d2\",\"parent\":null,\"primary_maintainer\":\"22f87e87-7fd9-4716-9dc6-e790372cac4c\",\"slug\":\"gfz\",\"name\":\"Helmholtz Centre Potsdam GFZ German Research Centre for Geosciences\",\"ror_id\":\"https://ror.org/04z8jg394\",\"website\":\"https://www.gfz-potsdam.de\",\"is_tenant\":true,\"created_at\":\"2022-07-26T19:34:55.173342+00:00\",\"updated_at\":\"2023-05-17T06:11:51.338233+00:00\",\"description\":\"# About GFZ Potsdam\n\nThe GFZ is Germany’s national research center for the solid Earth Sciences. Our mission is to deepen the knowledge of the dynamics of the solid Earth, and to develop solutions for grand challenges facing society. These challenges include anticipating the hazards arising from the Earth’s dynamic systems and mitigating the associated risks to society; securing our habitat under the pressure of global change; and supplying energy and mineral resources for a rapidly growing population in a sustainable manner and without harming the environment. \n\n## Research Software at GFZ\n\nTo learn more about the research software policy at GFZ, visit [gfz-potsdam.de/en/software](https://www.gfz-potsdam.de/en/software).\n\n\",\"logo_id\":\"da40b40113fdfb1d3ce40f58ecb784fe33ba40bb\",\"country\":null,\"city\":null,\"short_description\":null}]"; Collection result = RorPostgrestConnector.parseBasicJsonData(data); assertEquals(1, result.size()); diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorScraperTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorScraperTest.java index f6b1645..80bbe82 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorScraperTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ror/RorScraperTest.java @@ -1,46 +1,386 @@ +// SPDX-FileCopyrightText: 2024 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2026 Netherlands eScience Center // SPDX-FileCopyrightText: 2024 Christian Meeßen (GFZ) -// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) // SPDX-FileCopyrightText: 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences -// SPDX-FileCopyrightText: 2024 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.ror; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - +import nl.esciencecenter.rsd.scraper.RsdParsingException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; class RorScraperTest { + @Test + void givenInvalidJson_whenParsing_thenExceptionThrown() { + Assertions.assertThrowsExactly(RsdParsingException.class, () -> RorScraper.parseV2Data(null)); + Assertions.assertThrowsExactly(RsdParsingException.class, () -> RorScraper.parseV2Data("[]")); + Assertions.assertThrowsExactly(RsdParsingException.class, () -> RorScraper.parseV2Data("{")); + Assertions.assertThrowsExactly(RsdParsingException.class, () -> + RorScraper.parseV2Data("{\"errors\":[\"'test' is not a valid ROR ID\"]}") + ); + Assertions.assertThrowsExactly(RsdParsingException.class, () -> + RorScraper.parseV2Data("{\"errors\":[\"ROR ID 'https://ror.org/04tsk2645' does not exist\"]}") + ); + } + // example from https://github.com/ror-community/ror-schema/blob/master/example_record_v2_1.json @Test - void testLocations() { - String completeJsonResponse = "{\"id\":\"https://ror.org/04z8jg394\",\"name\":\"Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences\",\"email_address\":\"\",\"ip_addresses\":[],\"established\":1992,\"types\":[\"Facility\"],\"relationships\":[{\"label\":\"Helmholtz Association of German Research Centres\",\"type\":\"Parent\",\"id\":\"https://ror.org/0281dp749\"}],\"addresses\":[{\"lat\":52.39886,\"lng\":13.06566,\"state\":null,\"state_code\":null,\"city\":\"Potsdam\",\"geonames_city\":{\"id\":2852458,\"city\":\"Potsdam\",\"geonames_admin1\":{\"name\":\"Brandenburg\",\"id\":2945356,\"ascii_name\":\"Brandenburg\",\"code\":\"DE.11\"},\"geonames_admin2\":{\"name\":null,\"id\":null,\"ascii_name\":null,\"code\":\"DE.11.00\"},\"license\":{\"attribution\":\"Data from geonames.org under a CC-BY 3.0 license\",\"license\":\"http://creativecommons.org/licenses/by/3.0/\"},\"nuts_level1\":{\"name\":null,\"code\":null},\"nuts_level2\":{\"name\":null,\"code\":null},\"nuts_level3\":{\"name\":null,\"code\":null}},\"postcode\":null,\"primary\":false,\"line\":null,\"country_geonames_id\":2921044}],\"links\":[\"https://www.gfz-potsdam.de\"],\"aliases\":[],\"acronyms\":[\"GFZ\"],\"status\":\"active\",\"wikipedia_url\":\"https://en.wikipedia.org/wiki/GFZ_German_Research_Centre_for_Geosciences\",\"labels\":[{\"label\":\"Helmholtz-Zentrum Potsdam - Deutsches GeoForschungsZentrum GFZ\",\"iso639\":\"de\"}],\"country\":{\"country_name\":\"Germany\",\"country_code\":\"DE\"},\"external_ids\":{\"ISNI\":{\"preferred\":null,\"all\":[\"0000 0000 9195 2461\"]},\"FundRef\":{\"preferred\":\"501100010956\",\"all\":[\"501100010956\"]},\"Wikidata\":{\"preferred\":null,\"all\":[\"Q1205654\"]},\"GRID\":{\"preferred\":\"grid.23731.34\",\"all\":\"grid.23731.34\"}}}"; - RorData rorData = RorScraper.parseData(completeJsonResponse); + void givenExampleJson_whenParsing_thenCorrectDataReturned() { + // editorconfig-checker-disable + String exampleJson = """ + { + "locations": [ + { + "geonames_id": 5378538, + "geonames_details": { + "continent_code": "NA", + "continent_name": "North America", + "country_code": "US", + "country_name": "United States", + "country_subdivision_code": "CA", + "country_subdivision_name": "California", + "lat": 37.802168, + "lng": -122.271281, + "name": "Oakland" + } + } + ], + "established": 1868, + "external_ids": [ + { + "type": "fundref", + "all": [ + "100005595", + "100009350", + "100004802", + "100010574", + "100005188", + "100005192" + ], + "preferred": "100005595" + }, + { + "type": "grid", + "all": [ + "grid.30389.31" + ], + "preferred": "grid.30389.31" + }, + { + "type": "isni", + "all": [ + "0000 0001 2348 0690" + ], + "preferred": null + } + ], + "id": "https://ror.org/00pjdza24", + "domains": [ + "universityofcalifornia.edu" + ], + "links": [ + { + "type": "website", + "value": "http://www.universityofcalifornia.edu/" + }, + { + "type": "wikipedia", + "value": "http://en.wikipedia.org/wiki/University_of_California" + } + ], + "names": [ + { + "value": "UC", + "types": [ + "acronym" + ], + "lang": "en" + }, + { + "value": "UC System", + "types": [ + "alias" + ], + "lang": "en" + }, + { + "value": "University of California System", + "types": [ + "ror_display", + "label" + ], + "lang": "en" + }, + { + "value": "Université de Californie", + "types": [ + "label" + ], + "lang": "fr" + } + ], + "relationships": [ + { + "id": "https://ror.org/02jbv0t02", + "label": "Lawrence Berkeley National Laboratory", + "type": "related" + }, + { + "id": "https://ror.org/03yrm5c26", + "label": "California Digital Library", + "type": "child" + }, + { + "id": "https://ror.org/00zv0wd17", + "label": "Center for Information Technology Research in the Interest of Society", + "type": "child" + }, + { + "id": "https://ror.org/03t0t6y08", + "label": "University of California Division of Agriculture and Natural Resources", + "type": "child" + }, + { + "id": "https://ror.org/01an7q238", + "label": "University of California, Berkeley", + "type": "child" + }, + { + "id": "https://ror.org/05rrcem69", + "label": "University of California, Davis", + "type": "child" + }, + { + "id": "https://ror.org/04gyf1771", + "label": "University of California, Irvine", + "type": "child" + }, + { + "id": "https://ror.org/046rm7j60", + "label": "University of California, Los Angeles", + "type": "child" + }, + { + "id": "https://ror.org/00d9ah105", + "label": "University of California, Merced", + "type": "child" + }, + { + "id": "https://ror.org/03nawhv43", + "label": "University of California, Riverside", + "type": "child" + }, + { + "id": "https://ror.org/0168r3w48", + "label": "University of California, San Diego", + "type": "child" + }, + { + "id": "https://ror.org/043mz5j54", + "label": "University of California, San Francisco", + "type": "child" + }, + { + "id": "https://ror.org/02t274463", + "label": "University of California, Santa Barbara", + "type": "child" + }, + { + "id": "https://ror.org/03s65by71", + "label": "University of California, Santa Cruz", + "type": "child" + } + ], + "status": "active", + "types": [ + "education" + ], + "admin": { + "created": { + "date": "2020-04-25", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2022-10-18", + "schema_version": "2.0" + } + } + }"""; + // editorconfig-checker-enable - assertEquals("Potsdam", rorData.city()); - assertEquals("Germany", rorData.country()); - assertEquals(List.of("Facility"), rorData.rorTypes()); + RorData data = Assertions.assertDoesNotThrow(() -> RorScraper.parseV2Data(exampleJson)); + Assertions.assertEquals("United States", data.country()); + Assertions.assertEquals("Oakland", data.city()); + Assertions.assertEquals("http://en.wikipedia.org/wiki/University_of_California", data.wikipediaUrl()); + Assertions.assertEquals(List.of("education"), data.rorTypes()); + Assertions.assertEquals( + List.of("UC", "UC System", "University of California System", "Université de Californie"), + data.rorNames() + ); + Assertions.assertEquals(37.802168, data.lat()); + Assertions.assertEquals(-122.271281, data.lon()); } - @ParameterizedTest - @ValueSource(strings = { - "{\"addresses\": [{\"city\": null}], \"country\": {\"country_name\": null}}", - "{\"addresses\": [],\"country\": {}}", - "{}", - }) - void testNullLocationsOrEmptyLocationOrEmptyResponse(String jsonBody) { - RorData rorData = RorScraper.parseData(jsonBody); - - assertNull(rorData.city()); - assertNull(rorData.country()); + @Test + void givenHelmholtzJson_whenParsing_thenCorrectDataReturned() { + // editorconfig-checker-disable + String exampleJson = """ + { + "admin": { + "created": { + "date": "2018-11-14", + "schema_version": "1.0" + }, + "last_modified": { + "date": "2025-03-26", + "schema_version": "2.1" + } + }, + "domains": [ + "gfz-potsdam.de", + "gfz.de" + ], + "established": 1992, + "external_ids": [ + { + "all": [ + "501100010956" + ], + "preferred": "501100010956", + "type": "fundref" + }, + { + "all": [ + "grid.23731.34" + ], + "preferred": "grid.23731.34", + "type": "grid" + }, + { + "all": [ + "0000 0000 9195 2461" + ], + "preferred": null, + "type": "isni" + }, + { + "all": [ + "Q1205654" + ], + "preferred": null, + "type": "wikidata" + } + ], + "id": "https://ror.org/04z8jg394", + "links": [ + { + "type": "website", + "value": "https://www.gfz.de" + }, + { + "type": "wikipedia", + "value": "https://en.wikipedia.org/wiki/GFZ_Helmholtz_Centre_for_Geosciences" + } + ], + "locations": [ + { + "geonames_details": { + "continent_code": "EU", + "continent_name": "Europe", + "country_code": "DE", + "country_name": "Germany", + "country_subdivision_code": "BB", + "country_subdivision_name": "Brandenburg", + "lat": 52.39886, + "lng": 13.06566, + "name": "Potsdam" + }, + "geonames_id": 2852458 + } + ], + "names": [ + { + "lang": null, + "types": [ + "acronym" + ], + "value": "GFZ" + }, + { + "lang": "en", + "types": [ + "label", + "ror_display" + ], + "value": "GFZ Helmholtz Centre for Geosciences" + }, + { + "lang": "de", + "types": [ + "label" + ], + "value": "GFZ Helmholtz-Zentrum für Geoforschung" + }, + { + "lang": "en", + "types": [ + "alias" + ], + "value": "Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences" + }, + { + "lang": "de", + "types": [ + "alias" + ], + "value": "Helmholtz-Zentrum Potsdam - Deutsches GeoForschungsZentrum GFZ" + } + ], + "relationships": [ + { + "label": "Research Institute for Sustainability at GFZ", + "type": "child", + "id": "https://ror.org/01vvnmw35" + }, + { + "label": "Helmholtz Association of German Research Centres", + "type": "parent", + "id": "https://ror.org/0281dp749" + } + ], + "status": "active", + "types": [ + "facility", + "funder" + ] + }"""; + // editorconfig-checker-enable + + RorData data = Assertions.assertDoesNotThrow(() -> RorScraper.parseV2Data(exampleJson)); + + Assertions.assertEquals("Germany", data.country()); + Assertions.assertEquals("Potsdam", data.city()); + Assertions.assertEquals( + "https://en.wikipedia.org/wiki/GFZ_Helmholtz_Centre_for_Geosciences", + data.wikipediaUrl() + ); + Assertions.assertEquals(List.of("facility", "funder"), data.rorTypes()); + Assertions.assertEquals( + List.of( + "GFZ", + "GFZ Helmholtz Centre for Geosciences", + "GFZ Helmholtz-Zentrum für Geoforschung", + "Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences", + "Helmholtz-Zentrum Potsdam - Deutsches GeoForschungsZentrum GFZ" + ), + data.rorNames() + ); + Assertions.assertEquals(52.39886, data.lat()); + Assertions.assertEquals(13.06566, data.lon()); } }