diff --git a/.github/workflows/_ghcr.yml b/.github/workflows/_ghcr.yml index 1d05361..774f62e 100644 --- a/.github/workflows/_ghcr.yml +++ b/.github/workflows/_ghcr.yml @@ -51,15 +51,22 @@ jobs: run: | IMAGE_TAG_VERSION=${{inputs.base_image_name}}:${{inputs.image_tag}} IMAGE_TAG_LASTEST=${{inputs.base_image_name}}:latest - echo image_tag_version $IMAGE_TAG_VERSION - docker build -t $IMAGE_TAG_VERSION -t $IMAGE_TAG_LASTEST -f ${{inputs.dockerfile}} ${{inputs.docker_context}} - echo "{image_build}={$IMAGE_TAG_VERSIONvalue}" >> $GITHUB_OUTPUT + + echo "Build image: $IMAGE_TAG_VERSION and pass APP_VERSION=${{inputs.image_tag}}" + + echo "docker build -t $IMAGE_TAG_VERSION -t $IMAGE_TAG_LATEST --build-arg APP_VERSION=${{inputs.image_tag}} -f ${{inputs.dockerfile}} ${{inputs.docker_context}}" + docker build -t $IMAGE_TAG_VERSION -t $IMAGE_TAG_LATEST --build-arg APP_VERSION=${{inputs.image_tag}} -f ${{inputs.dockerfile}} ${{inputs.docker_context}} + + echo "image_build=$IMAGE_TAG_VERSION" >> $GITHUB_OUTPUT + - name: push to ghcr.io id: push_image run: | - echo login + echo "Logging in to GHCR..." + echo "${{secrets.token}}" | docker login https://ghcr.io -u ${{inputs.ghcr_user}} --password-stdin echo push auth image with all tags docker push ${{inputs.base_image_name}} --all-tags - echo "{image_build}={$IMAGE_TAG_VERSIONvalue}" >> $GITHUB_OUTPUT + + echo "image_pushed=true" >> $GITHUB_OUTPUT diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9739d69..87e0b73 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,6 +59,66 @@ jobs: echo skipped=${{needs.release_tag.outputs.skipped}} echo tag=${{needs.release_tag.outputs.tag}} + auth: + # it needs to be checked on string value + if: needs.release_tag.outputs.skipped == 'false' + needs: release_tag + name: auth + uses: ./.github/workflows/_ghcr.yml + with: + ghcr_user: ${{github.actor}} + base_image_name: ghcr.io/research-software-directory/kin-rpd/auth + image_tag: ${{needs.release_tag.outputs.tag}} + dockerfile: authentication/Dockerfile + docker_context: ./authentication + secrets: + token: ${{secrets.GITHUB_TOKEN}} + + database: + # it needs to be checked on string value + if: needs.release_tag.outputs.skipped == 'false' + needs: release_tag + name: database + uses: ./.github/workflows/_ghcr.yml + with: + ghcr_user: ${{github.actor}} + base_image_name: ghcr.io/research-software-directory/kin-rpd/database + image_tag: ${{needs.release_tag.outputs.tag}} + dockerfile: database/Dockerfile + docker_context: ./database + secrets: + token: ${{secrets.GITHUB_TOKEN}} + + backend: + # it needs to be checked on string value + if: needs.release_tag.outputs.skipped == 'false' + needs: release_tag + name: backend api + uses: ./.github/workflows/_ghcr.yml + with: + ghcr_user: ${{github.actor}} + base_image_name: ghcr.io/research-software-directory/rsd-saas/backend + image_tag: ${{needs.release_tag.outputs.tag}} + dockerfile: backend-postgrest/Dockerfile + docker_context: ./backend-postgrest + secrets: + token: ${{secrets.GITHUB_TOKEN}} + + background-services: + # it needs to be checked on string value + if: needs.release_tag.outputs.skipped == 'false' + needs: release_tag + name: background services + uses: ./.github/workflows/_ghcr.yml + with: + ghcr_user: ${{github.actor}} + base_image_name: ghcr.io/research-software-directory/kin-rpd/background-services + image_tag: ${{needs.release_tag.outputs.tag}} + dockerfile: background-services/Dockerfile + docker_context: ./background-services + secrets: + token: ${{secrets.GITHUB_TOKEN}} + frontend: # it needs to be checked on string value if: needs.release_tag.outputs.skipped == 'false' diff --git a/Makefile b/Makefile index 4b3fd8e..247731e 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ export DGID # ---------------------------------------------------------------- start: clean docker compose build # build all services - docker compose up --scale data-generation=1 --scale scrapers=0 --detach + docker compose up --scale data-generation=1 --detach # open http://localhost to see the application running install: clean diff --git a/background-services/Dockerfile b/background-services/Dockerfile deleted file mode 100644 index ad54d2f..0000000 --- a/background-services/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) -# SPDX-FileCopyrightText: 2025 Netherlands eScience Center -# -# SPDX-License-Identifier: Apache-2.0 - -FROM maven:3.10.0-eclipse-temurin-25-noble AS builder -WORKDIR /usr/mymaven -RUN mkdir src -COPY pom.xml . -RUN mvn dependency:go-offline -COPY ./src ./src -RUN mvn package --offline - -FROM eclipse-temurin:25.0.4.1_1-jre-noble -WORKDIR /usr/myjava -COPY --from=builder /usr/mymaven/target/*-jar-with-dependencies.jar background-services.jar -CMD java -cp background-services.jar nl.esciencecenter.rsd.MainDatabase diff --git a/background-services/README.md b/background-services/README.md deleted file mode 100644 index 34d6af3..0000000 --- a/background-services/README.md +++ /dev/null @@ -1,20 +0,0 @@ - - -# Background services module - -This module currently auto-updates the materialized view `count_software_mentions_cached` every five minutes. Look in `.env.example` (search for `background-services`) to see what environment variables need to be set. - -## Developing - -To run the program locally, remember to load in the env variables and override the value of `POSTGRES_DB_HOST` to `localhost`. - -Please respect the current code style (e.g. tabs for indentation) and prevent your IDE/editor to make style changes to existing code. The code is currently formatted using IntelliJ IDEA. - -## Future possibilities - -This module could be expanded to allow admins to schedule arbitrary database queries, e.g. to auto-clean dangling images and mentions. diff --git a/background-services/pom.xml b/background-services/pom.xml deleted file mode 100644 index 96c092d..0000000 --- a/background-services/pom.xml +++ /dev/null @@ -1,83 +0,0 @@ - - - - - - 4.0.0 - - nl.esciencecenter - background-services - 1.0-SNAPSHOT - - - 25 - 25 - UTF-8 - research-software-directory - https://sonarcloud.io - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 3.8.1 - - - - - org.apache.maven.plugins - maven-assembly-plugin - 3.8.0 - - - - jar-with-dependencies - - - - - - package - - single - - - - - - - - - - - org.postgresql - postgresql - 42.7.13 - - - - - org.slf4j - slf4j-api - 2.0.18 - - - - - ch.qos.logback - logback-classic - 1.6.5 - - - - diff --git a/background-services/src/main/java/nl/esciencecenter/rsd/MainDatabase.java b/background-services/src/main/java/nl/esciencecenter/rsd/MainDatabase.java deleted file mode 100644 index 449b000..0000000 --- a/background-services/src/main/java/nl/esciencecenter/rsd/MainDatabase.java +++ /dev/null @@ -1,57 +0,0 @@ -// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2025 Netherlands eScience Center -// -// SPDX-License-Identifier: Apache-2.0 - -package nl.esciencecenter.rsd; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -public class MainDatabase { - - private static final String VIEW_NAME = "count_software_mentions_cached"; - - public static void main(String[] args) { - Logger logger = LoggerFactory.getLogger(MainDatabase.class); - - Runnable runnable = () -> { - String dbHost = System.getenv("POSTGRES_DB_HOST"); - String dbPort = System.getenv("POSTGRES_DB_HOST_PORT"); - String dbName = System.getenv("POSTGRES_DB"); - String dbUrl = "jdbc:postgresql://%s:%s/%s".formatted(dbHost, dbPort, dbName); - String dbUsername = System.getenv("POSTGRES_USER"); - String dbPassword = System.getenv("POSTGRES_PASSWORD"); - try ( - Connection connection = DriverManager.getConnection(dbUrl, dbUsername, dbPassword); - Statement statement = connection.createStatement() - ) { - statement.execute("REFRESH MATERIALIZED VIEW CONCURRENTLY %s;".formatted(VIEW_NAME)); - } catch (SQLException e) { - throw new RuntimeException(e); - } - }; - - try (ScheduledExecutorService runner = Executors.newSingleThreadScheduledExecutor()) { - int initialDelay = 30; - int delay = 300; - ScheduledFuture future = runner.scheduleWithFixedDelay(runnable, initialDelay, delay, TimeUnit.SECONDS); - future.get(); - } catch (ExecutionException e) { - logger.error("Something went wrong updating the materialized view \"%s\"".formatted(VIEW_NAME), e); - } catch (InterruptedException e) { - logger.error("Got interrupted when updating the materialized view \"%s\"".formatted(VIEW_NAME), e); - Thread.currentThread().interrupt(); - } - } -} diff --git a/background-services/src/main/resources/logback.xml b/background-services/src/main/resources/logback.xml deleted file mode 100644 index 1030ab4..0000000 --- a/background-services/src/main/resources/logback.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} -%kvp- %msg%n - - - - - - - diff --git a/data-generation/package-lock.json b/data-generation/package-lock.json index ec7a19a..5c48bee 100644 --- a/data-generation/package-lock.json +++ b/data-generation/package-lock.json @@ -389,9 +389,9 @@ } }, "node_modules/joi": { - "version": "18.3.0", - "resolved": "https://registry.npmjs.org/joi/-/joi-18.3.0.tgz", - "integrity": "sha512-/RwAK4BAfzcdbdFg073tzq4/xhY6Q7lMd5AmjqTL6pu0QFJdEmtswgm8BvgwXvOKuPNYQ8XrdQrrW/gF65LiNw==", + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/joi/-/joi-18.3.1.tgz", + "integrity": "sha512-mV9M4Llh4va3NT90p7KBtQhoZqjs8xLX7EWYE4y/GG3yLF6REbIBEHSUb0UX1JXRht2w5tiLSsvmpYc5akiqIw==", "license": "BSD-3-Clause", "dependencies": { "@hapi/address": "^5.1.1", diff --git a/deployment/.env.example b/deployment/.env.example index ff0df34..c867b61 100644 --- a/deployment/.env.example +++ b/deployment/.env.example @@ -123,12 +123,12 @@ LINKEDIN_WELL_KNOWN_URL=https://www.linkedin.com/oauth/.well-known/openid-config # max requests to the GitHub API per run, runs 10 times per hour # optional, comment out if not available, a default of 6 will be used # consumed by: scrapers -MAX_REQUESTS_GITHUB=6 +MAX_REQUESTS_GITHUB=0 # max request to GitLab API per run, runs 10 times per hour # optional, comment out if not available, a default of 6 will be used # consumed by: scrapers -MAX_REQUESTS_GITLAB=6 +MAX_REQUESTS_GITLAB=0 # max mentions to scrape per run, runs 10 times per hour # optional, comment out if not available, a default of 6 will be used @@ -140,6 +140,11 @@ MAX_REQUESTS_DOI=6 # consumed by: scrapers MAX_REQUESTS_ROR=6 +# max reference papers of which to scrape citations per run, runs 10 times per hour +# optional, comment out if not available, a default of 6 will be used +# consumed by: scrapers +MAX_REQUESTS_OPENALEX=6 + # ---- SECRETS ------ SECRETS ----------- # consumed by services: database @@ -194,3 +199,6 @@ MATOMO_ID= # consumed by: scrapers # LIBRARIES_IO_ACCESS_TOKEN= + +# See https://developers.openalex.org/guides/authentication for more information +OPENALEX_API_KEY= diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index 21e2246..d7a5c52 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -5,7 +5,7 @@ services: database: container_name: database - image: ghcr.io/research-software-directory/rsd-saas/database:v3.4.1 + image: ghcr.io/research-software-directory/kin-rpd/database:latest expose: - 5432 environment: @@ -19,13 +19,19 @@ services: # to remove use: docker compose down --volumes # to inspect use: docker volume ls - pgdb:/var/lib/postgresql/data/ + healthcheck: + test: ["CMD", "pg_isready", "--username=${POSTGRES_USER}", "--dbname=${POSTGRES_DB}"] + interval: 1s + retries: 15 + timeout: 5s + start_period: 5s networks: - net restart: unless-stopped backend: container_name: backend - image: ghcr.io/research-software-directory/rsd-saas/backend:v3.4.1 + image: ghcr.io/research-software-directory/kin-rpd/backend:latest expose: - 3500 environment: @@ -36,14 +42,15 @@ services: - PGRST_SERVER_PORT - PGRST_JWT_SECRET depends_on: - - database + database: + condition: service_healthy networks: - net restart: unless-stopped auth: container_name: auth - image: ghcr.io/research-software-directory/rsd-saas/auth:v3.4.1 + image: ghcr.io/research-software-directory/kin-rpd/auth:latest expose: - 7000 environment: @@ -81,48 +88,44 @@ services: - AUTH_LINKEDIN_CLIENT_SECRET - PGRST_JWT_SECRET depends_on: - - database - - backend + database: + condition: service_healthy + backend: + condition: service_started networks: - net restart: unless-stopped scrapers: container_name: scrapers - image: ghcr.io/research-software-directory/rsd-saas/scrapers:v3.4.1 + image: ghcr.io/research-software-directory/kin-rpd/scrapers:latest environment: # it uses values from .env file - POSTGREST_URL - - PGRST_JWT_SECRET - MAX_REQUESTS_GITHUB - MAX_REQUESTS_GITLAB - - API_CREDENTIALS_GITHUB + - MAX_REQUESTS_4TU + - MAX_REQUESTS_CODEBERG + - MAX_REQUESTS_DOI + - MAX_REQUESTS_OPENALEX - MAX_REQUESTS_ROR + - ENABLE_NASSA_SCRAPER + - ENABLE_ESCAPE_OSSR_SCRAPER + - PGRST_JWT_SECRET + - API_CREDENTIALS_GITHUB - ZENODO_ACCESS_TOKEN - - MAX_REQUESTS_DOI - CROSSREF_CONTACT_EMAIL + - OPENALEX_API_KEY - LIBRARIES_IO_ACCESS_TOKEN depends_on: - - database - - backend + database: + condition: service_healthy + backend: + condition: service_started networks: - net restart: unless-stopped - background-services: - image: ghcr.io/research-software-directory/rsd-saas/background-services:v3.4.1 - environment: - # it uses values from .env file - - POSTGRES_DB_HOST - - POSTGRES_DB_HOST_PORT - - POSTGRES_DB - - POSTGRES_USER - - POSTGRES_PASSWORD - depends_on: - - database - networks: - - net - frontend: container_name: frontend image: ghcr.io/research-software-directory/kin-rpd/frontend:latest @@ -150,9 +153,12 @@ services: expose: - 3000 depends_on: - - database - - backend - - auth + database: + condition: service_healthy + backend: + condition: service_started + auth: + condition: service_started networks: - net restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index 1959186..3b908d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,12 @@ services: # to remove use: docker compose down --volumes # to inspect use: docker volume ls - pgdb:/var/lib/postgresql/data/ + healthcheck: + test: ["CMD", "pg_isready", "--username=${POSTGRES_USER}", "--dbname=${POSTGRES_DB}"] + interval: 1s + retries: 15 + timeout: 5s + start_period: 5s networks: - net @@ -40,24 +46,8 @@ services: - PGRST_SERVER_PORT - PGRST_JWT_SECRET depends_on: - - database - networks: - - net - - background-services: - build: ./backend-postgrest - image: kin-rpd/background-services:v1.1.2 - # initial rsd image version - # image: ghcr.io/research-software-directory/rsd-saas/background-services:v3.4.1 - environment: - # it uses values from .env file - - POSTGRES_DB_HOST - - POSTGRES_DB_HOST_PORT - - POSTGRES_DB - - POSTGRES_USER - - POSTGRES_PASSWORD - depends_on: - - database + database: + condition: service_healthy networks: - net @@ -105,8 +95,10 @@ services: - AUTH_LINKEDIN_CLIENT_SECRET - PGRST_JWT_SECRET depends_on: - - database - - backend + database: + condition: service_healthy + backend: + condition: service_started networks: - net entrypoint: @@ -121,22 +113,30 @@ services: build: ./scrapers image: kin-rpd/scrapers:v1.1.2 # initial rsd image version - # image: ghcr.io/research-software-directory/rsd-saas/scrapers:v3.4.1 + # image: ghcr.io/research-software-directory/rsd-saas/scrapers:v6.7.5 environment: # it uses values from .env file - POSTGREST_URL - - PGRST_JWT_SECRET - MAX_REQUESTS_GITHUB - MAX_REQUESTS_GITLAB + - MAX_REQUESTS_4TU + - MAX_REQUESTS_CODEBERG + - MAX_REQUESTS_DOI + - MAX_REQUESTS_OPENALEX - MAX_REQUESTS_ROR + - ENABLE_NASSA_SCRAPER + - ENABLE_ESCAPE_OSSR_SCRAPER + - PGRST_JWT_SECRET - API_CREDENTIALS_GITHUB - ZENODO_ACCESS_TOKEN - - MAX_REQUESTS_DOI - CROSSREF_CONTACT_EMAIL + - OPENALEX_API_KEY - LIBRARIES_IO_ACCESS_TOKEN depends_on: - - database - - backend + database: + condition: service_healthy + backend: + condition: service_started networks: - net @@ -145,6 +145,10 @@ services: context: ./frontend # dockerfile to use for build dockerfile: Dockerfile + # rsd version passed into image + # for release the tag is passed in GH action _ghcr.yml + args: + APP_VERSION: "1.1.2" # update version number to correspond to frontend/package.json image: kin-rpd/frontend:1.1.2 environment: @@ -171,9 +175,12 @@ services: expose: - 3000 depends_on: - - database - - backend - - auth + database: + condition: service_healthy + backend: + condition: service_started + auth: + condition: service_started volumes: - ./frontend/public:/app/public # volumes: @@ -224,7 +231,10 @@ services: - PGRST_JWT_SECRET - POSTGREST_URL depends_on: - - backend + database: + condition: service_healthy + backend: + condition: service_started networks: - net deploy: diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 485e733..31beb58 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -59,7 +59,7 @@ ENV NODE_ENV=production # Pass version as build argument ARG APP_VERSION=unknown -ENV RSD_VERSION=$APP_VERSION +ENV KIN_VERSION=$APP_VERSION RUN addgroup --system --gid 1001 nodejs RUN adduser --system --uid 1001 nextjs diff --git a/frontend/pages/api/fe/index.ts b/frontend/pages/api/fe/index.ts index c7cf489..db35db9 100644 --- a/frontend/pages/api/fe/index.ts +++ b/frontend/pages/api/fe/index.ts @@ -9,7 +9,8 @@ import app from '../../../package.json' type Data = { module: string, - version: string, + kin_version: string, + fe_version: string, status: string } @@ -17,5 +18,10 @@ export default function handler( req: NextApiRequest, res: NextApiResponse ) { - res.status(200).json({module: 'frontend/api', version: app.version, status:'live'}) + res.status(200).json({ + module: 'frontend/api', + kin_version: process.env.KIN_VERSION || 'MISSING', + fe_version: app.version, + status:'live' + }) } diff --git a/scrapers/.editorconfig b/scrapers/.editorconfig index 76769f3..7dda6f1 100644 --- a/scrapers/.editorconfig +++ b/scrapers/.editorconfig @@ -1,5 +1,5 @@ -# SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) -# SPDX-FileCopyrightText: 2024 Netherlands eScience Center +# SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center # # SPDX-License-Identifier: Apache-2.0 @@ -15,6 +15,3 @@ indent_style = tab [**.md] indent_style = unset - -[**.py] -indent_style = unset diff --git a/scrapers/Dockerfile b/scrapers/Dockerfile index 1194123..e582670 100644 --- a/scrapers/Dockerfile +++ b/scrapers/Dockerfile @@ -1,5 +1,5 @@ -# SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) -# SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +# SPDX-FileCopyrightText: 2022 - 2026 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2022 - 2026 Netherlands eScience Center # SPDX-FileCopyrightText: 2022 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences # SPDX-FileCopyrightText: 2022 Matthias Rüster (GFZ) # diff --git a/scrapers/README.md b/scrapers/README.md index b6d0171..ae5c99c 100644 --- a/scrapers/README.md +++ b/scrapers/README.md @@ -1,6 +1,6 @@ @@ -29,8 +29,39 @@ docker compose exec scrapers java -cp /usr/myjava/scrapers.jar nl.esciencecenter docker compose exec scrapers java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.package_manager.MainPackageManager docker compose exec scrapers java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.doi.MainCitations docker compose exec scrapers java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.ror.MainRor +docker compose exec scrapers java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.aggregator.MainAggregator ``` ## Running locally To run a scraper locally, e.g. from an IDE, you need to make sure the values in `.env` are loaded and that the value for the env variable `POSTGREST_URL` is overwritten with an address that is reachable from outside of the Docker network (usually `http://localhost/api/v1`). + +### NASSA harvester + +**This harvester is only for the production instance of the Netherlands eScience Center.** + +This harvester harvests and stores the [NASSA modules](https://github.com/Archaeology-ABM/NASSA-modules). For it to run, it is required that there is a community with slug `nassa`. + +It is scheduled to run once per day (see `jobs.cron`). + +It can manually be activated with: + +```shell +docker compose exec scrapers java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.nassa.MainNassa +``` + +### ESCAPE OSSR harvester + +**This harvester is only for the production instance of the Netherlands eScience Center.** + +This harvester harvests and stores the [ESCAPE OSSR software](https://zenodo.org/communities/escape2020/records?q=&f=resource_type%3Asoftware&l=list&p=1&s=10&sort=newest). For it to run, it is required that there is a community with slug `escape-ossr`. + +It is scheduled to run once per day (see `jobs.cron`). + +It can manually be activated with: + +```shell +docker compose exec scrapers java -cp /usr/myjava/scrapers.jar --enable-native-access=ALL-UNNAMED nl.esciencecenter.rsd.scraper.escape_ossr.MainEscapeOssr +``` + +The `--enable-native-access=ALL-UNNAMED` flag is to allow the [html-to-markdown](https://docs.html-to-markdown.xberg.io/) to run without warnings. diff --git a/scrapers/jobs.cron b/scrapers/jobs.cron index 6645377..3978bcb 100644 --- a/scrapers/jobs.cron +++ b/scrapers/jobs.cron @@ -2,14 +2,18 @@ # # SPDX-License-Identifier: CC-BY-4.0 # -*/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainProgrammingLanguages > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 -*/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.package_manager.MainPackageManager > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 -2-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainBasicData > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 -4-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainCommits > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 -1-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.doi.MainReleases > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +# Use ONLY RELEVANT KIN scrapers for project/mentions + +# */6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainProgrammingLanguages > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +# */6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.package_manager.MainPackageManager > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +# 2-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainBasicData > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +# 4-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainCommits > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +# 1-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.doi.MainReleases > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +# 5-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainContributors > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +# 15 1 * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.nassa.MainNassa > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 +# 25 1 * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar --enable-native-access=ALL-UNNAMED nl.esciencecenter.rsd.scraper.escape_ossr.MainEscapeOssr > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 + 3-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.doi.MainMentions > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 -5-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.git.MainContributors > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 5-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.doi.MainCitations > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 2-59/6 * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.ror.MainRor > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 * * * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.aggregator.MainAggregator > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 -15 1 * * * /opt/java/openjdk/bin/java -cp /usr/myjava/scrapers.jar nl.esciencecenter.rsd.scraper.nassa.MainNassa > /proc/$(cat /var/run/crond.pid)/fd/1 2>&1 diff --git a/scrapers/pom.xml b/scrapers/pom.xml index fa58713..be0fdcb 100644 --- a/scrapers/pom.xml +++ b/scrapers/pom.xml @@ -3,8 +3,8 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 @@ -99,6 +99,7 @@ SPDX-License-Identifier: Apache-2.0 + org.junit junit-bom @@ -145,6 +146,14 @@ SPDX-License-Identifier: Apache-2.0 2.6 + + + io.xberg + html-to-markdown + 3.10.0 + compile + + diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/BibtexParser.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/BibtexParser.java index 8278683..e2e1a82 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/BibtexParser.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/BibtexParser.java @@ -5,16 +5,15 @@ package nl.esciencecenter.rsd.scraper; -import nl.esciencecenter.rsd.scraper.doi.Doi; -import nl.esciencecenter.rsd.scraper.doi.ExternalMentionRecord; -import nl.esciencecenter.rsd.scraper.doi.MentionType; - import java.io.IOException; import java.io.LineNumberReader; import java.io.Reader; import java.net.URI; import java.util.Map; import java.util.TreeMap; +import nl.esciencecenter.rsd.scraper.doi.Doi; +import nl.esciencecenter.rsd.scraper.doi.ExternalMentionRecord; +import nl.esciencecenter.rsd.scraper.doi.MentionType; // https://www.bibtex.org/Format/ // https://www.bibtex.com/e/entry-types/ @@ -52,7 +51,7 @@ enum ParserState { READING_MONTH, READING_INTEGER, READING_VALUE, - READING_COMMA + READING_COMMA, } public static Map parse(Reader reader) throws IOException, InvalidInputException { @@ -96,13 +95,24 @@ public static Map parse(Reader reader) throws IOE case READING_ENTRY_TYPE -> { if (Character.isWhitespace(c) || c == '{') { if (stringBuilder.isEmpty()) { - throw new InvalidInputException("READING_ENTRY_TYPE Empty entry type at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "READING_ENTRY_TYPE Empty entry type at line %d, column %d".formatted( + lineNumberReader.getLineNumber(), + column + ) + ); } entryType = stringBuilder.toString(); stringBuilder.setLength(0); state = c == '{' ? ParserState.BEFORE_READING_CITEKEY : ParserState.READING_ENTRY_OUTER_BRACE; } else if (!isAsciiLetter(c)) { - throw new InvalidInputException("READING_ENTRY_TYPE Expected a letter but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "READING_ENTRY_TYPE Expected a letter but got %c at line %d, column %d".formatted( + c, + lineNumberReader.getLineNumber(), + column + ) + ); } else { stringBuilder.append(c); } @@ -111,12 +121,24 @@ public static Map parse(Reader reader) throws IOE if (c == '{') { state = ParserState.BEFORE_READING_KEY; } else if (!Character.isWhitespace(c)) { - throw new InvalidInputException("READING_ENTRY_OUTER_BRACE Expected { or whitespace but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "READING_ENTRY_OUTER_BRACE Expected { or whitespace but got %c at line %d, column %d".formatted( + c, + lineNumberReader.getLineNumber(), + column + ) + ); } } case BEFORE_READING_CITEKEY -> { if (!isValidCitekeyCharacter(c)) { - throw new InvalidInputException("BEFORE_READING_CITEKEY Expected an alphanumeric character or - or _ or : but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "BEFORE_READING_CITEKEY Expected an alphanumeric character or - or _ or : but got %c at line %d, column %d".formatted( + c, + lineNumberReader.getLineNumber(), + column + ) + ); } else if (!Character.isWhitespace(c)) { stringBuilder.setLength(0); stringBuilder.append(c); @@ -128,13 +150,24 @@ public static Map parse(Reader reader) throws IOE stringBuilder.append(c); } else if (c == ',') { if (stringBuilder.isEmpty()) { - throw new InvalidInputException("READING_CITEKEY Empty citekey at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "READING_CITEKEY Empty citekey at line %d, column %d".formatted( + lineNumberReader.getLineNumber(), + column + ) + ); } citekey = stringBuilder.toString(); stringBuilder.setLength(0); state = ParserState.BEFORE_READING_KEY; } else { - throw new InvalidInputException("READING_CITEKEY Expected an alphanumeric character or - or _ or : but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "READING_CITEKEY Expected an alphanumeric character or - or _ or : but got %c at line %d, column %d".formatted( + c, + lineNumberReader.getLineNumber(), + column + ) + ); } } case BEFORE_READING_KEY -> { @@ -146,7 +179,13 @@ public static Map parse(Reader reader) throws IOE stringBuilder.append(c); state = ParserState.READING_KEY; } else if (!Character.isWhitespace(c)) { - throw new InvalidInputException("BEFORE_READING_KEY Expected a letter but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "BEFORE_READING_KEY Expected a letter but got %c at line %d, column %d".formatted( + c, + lineNumberReader.getLineNumber(), + column + ) + ); } } case READING_KEY -> { @@ -155,20 +194,37 @@ public static Map parse(Reader reader) throws IOE stringBuilder.append(c); } else if (Character.isWhitespace(c) || c == '=') { if (stringBuilder.isEmpty()) { - throw new InvalidInputException("READING_KEY Empty key at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "READING_KEY Empty key at line %d, column %d".formatted( + lineNumberReader.getLineNumber(), + column + ) + ); } key = stringBuilder.toString(); stringBuilder.setLength(0); state = c == '=' ? ParserState.READING_VALUE_DELIMITER : ParserState.READING_EQUALS_SIGN; } else { - throw new InvalidInputException("READING_KEY Expected a letter or whitespace or = but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "READING_KEY Expected a letter or whitespace or = but got %c at line %d, column %d".formatted( + c, + lineNumberReader.getLineNumber(), + column + ) + ); } } case READING_EQUALS_SIGN -> { if (c == '=') { state = ParserState.READING_VALUE_DELIMITER; } else if (Character.isWhitespace(c)) { - throw new InvalidInputException("READING_EQUALS_SIGN Expected whitespace or = but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "READING_EQUALS_SIGN Expected whitespace or = but got %c at line %d, column %d".formatted( + c, + lineNumberReader.getLineNumber(), + column + ) + ); } } case READING_VALUE_DELIMITER -> { @@ -187,7 +243,13 @@ public static Map parse(Reader reader) throws IOE stringBuilder.append(c); state = ParserState.READING_INTEGER; } else if (!Character.isWhitespace(c)) { - throw new InvalidInputException("READING_VALUE_DELIMITER Expected whitespace or { or \" but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "READING_VALUE_DELIMITER Expected whitespace or { or \" but got %c at line %d, column %d".formatted( + c, + lineNumberReader.getLineNumber(), + column + ) + ); } } case READING_MONTH -> { @@ -195,18 +257,33 @@ public static Map parse(Reader reader) throws IOE if (stringBuilder.length() < 3) { stringBuilder.append(c); } else { - throw new InvalidInputException("READING_MONTH Month literal too long at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "READING_MONTH Month literal too long at line %d, column %d".formatted( + lineNumberReader.getLineNumber(), + column + ) + ); } } else if (Character.isWhitespace(c) || c == ',') { if (stringBuilder.length() != 3) { - throw new InvalidInputException("READING_MONTH Month literal too short at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "READING_MONTH Month literal too short at line %d, column %d".formatted( + lineNumberReader.getLineNumber(), + column + ) + ); } else { keyValues.put(key, stringBuilder.toString()); stringBuilder.setLength(0); state = c == ',' ? ParserState.BEFORE_READING_KEY : ParserState.READING_COMMA; } } else { - throw new InvalidInputException("READING_MONTH Illegal month literal at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "READING_MONTH Illegal month literal at line %d, column %d".formatted( + lineNumberReader.getLineNumber(), + column + ) + ); } } case READING_INTEGER -> { @@ -217,7 +294,12 @@ public static Map parse(Reader reader) throws IOE stringBuilder.setLength(0); state = c == ',' ? ParserState.BEFORE_READING_KEY : ParserState.READING_COMMA; } else { - throw new InvalidInputException("READING_INTEGER Illegal integer literal at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "READING_INTEGER Illegal integer literal at line %d, column %d".formatted( + lineNumberReader.getLineNumber(), + column + ) + ); } } case READING_VALUE -> { @@ -226,7 +308,12 @@ public static Map parse(Reader reader) throws IOE } else if (c == '}') { --openBraceCount; if (openBraceCount < 0 && valueDelimiter != '{') { - throw new InvalidInputException("READING_VALUE Unmatched closing brace at line %d, column %d".formatted(lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "READING_VALUE Unmatched closing brace at line %d, column %d".formatted( + lineNumberReader.getLineNumber(), + column + ) + ); } else if (openBraceCount < 0) { keyValues.put(key, stringBuilder.toString()); stringBuilder.setLength(0); @@ -248,14 +335,23 @@ public static Map parse(Reader reader) throws IOE state = ParserState.BETWEEN_ENTRIES; handleEntry(keyValues, entryType, citekey, result); } else if (!Character.isWhitespace(c)) { - throw new InvalidInputException("READING_COMMA Expected whitespace or , or } but got %c at line %d, column %d".formatted(c, lineNumberReader.getLineNumber(), column)); + throw new InvalidInputException( + "READING_COMMA Expected whitespace or , or } but got %c at line %d, column %d".formatted( + c, + lineNumberReader.getLineNumber(), + column + ) + ); } } } } } - static Map handleEndOfInput(Map result, ParserState state) throws InvalidInputException { + static Map handleEndOfInput( + Map result, + ParserState state + ) throws InvalidInputException { if (state == ParserState.BETWEEN_ENTRIES) { return result; } else { @@ -264,17 +360,14 @@ static Map handleEndOfInput(Map= 'a' && c <= 'z') - || - (c >= 'A' && c <= 'Z') - || - (c >= '0' && c <= '9') - || - c == '-' - || - c == '_' - || - c == ':'; + return ( + (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '-' || + c == '_' || + c == ':' + ); } static boolean isAsciiLetter(char c) { @@ -332,4 +425,3 @@ static void handleEntry( keyValues.clear(); } } - diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Config.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Config.java index 87f9ca6..757a3c9 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Config.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Config.java @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2022 - 2024 Christian Meeßen (GFZ) -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) // SPDX-FileCopyrightText: 2022 - 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2026 Netherlands eScience Center // SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) // SPDX-FileCopyrightText: 2022 dv4all // @@ -9,11 +9,10 @@ package nl.esciencecenter.rsd.scraper; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Optional; - /** * Helper class to retrieve various environment variables. */ @@ -23,8 +22,7 @@ public class Config { private static final Logger LOGGER = LoggerFactory.getLogger(Config.class); - private Config() { - } + private Config() {} /** * Retrieves the value of the environment variable with the given name. Variable is not allowed to unset, and an Error @@ -34,7 +32,6 @@ private Config() { * @return the value of the variable. */ private static String getMandatoryEnv(String name) { - if (name == null || name.isBlank()) { LOGGER.error("Attempting to retrieve mandatory environment variable without a name!"); throw new Error("Attempting to retrieve mandatory environment variable without a name!"); @@ -64,7 +61,6 @@ private static String getMandatoryEnv(String name) { * @return the value of the variable if set, or empty Optional otherwise. */ private static Optional getOptionalEnv(String name) { - if (name == null || name.isBlank()) { LOGGER.warn("Attempting to retrieve environment variable without a name!"); return Optional.empty(); @@ -72,7 +68,7 @@ private static Optional getOptionalEnv(String name) { try { String value = System.getenv(name); - return (value == null || value.isBlank()) ? Optional.empty() : Optional.of(value.strip()); + return value == null || value.isBlank() ? Optional.empty() : Optional.of(value.strip()); } catch (Exception e) { LOGGER.warn("Failed to retrieve environment variable: {}", name, e); return Optional.empty(); @@ -88,7 +84,6 @@ private static Optional getOptionalEnv(String name) { * @return the value from the environment of the default value if no value could be obtained */ private static int getIntEnv(String name, int defaultValue) { - if (name == null || name.isBlank()) { LOGGER.warn("Attempting to retrieve environment variable without a name!"); return defaultValue; @@ -149,6 +144,24 @@ public static int maxRequestsGitLab() { return getIntEnv("MAX_REQUESTS_GITLAB", 6); } + /** + * The maximum requests rate for the 4TU Git API. + * + * @return the maximum request rate (default 6). + */ + public static int maxRequests4tu() { + return getIntEnv("MAX_REQUESTS_4TU", 6); + } + + /** + * The maximum requests rate for the Codeberg API. + * + * @return the maximum request rate (default 6). + */ + public static int maxRequestsCodeberg() { + return getIntEnv("MAX_REQUESTS_CODEBERG", 6); + } + /** * Get the maximum number of mentions to scrape in one run of the MentionScraper. * @@ -168,18 +181,27 @@ public static int maxRequestsRor() { * @return the maximum number of citation sources to scrape (default 5). */ public static int maxCitationSourcesToScrape() { - return getIntEnv("MAX_REQUESTS_OPENALEX", 5); + return getIntEnv("MAX_REQUESTS_OPENALEX", 6); } /** - * Get the email contact address for crossref. + * Get the email contact address for Crossref. * - * @return the Get the email contact address for crossref (default unset). + * @return the email contact address for Crossref (default unset). */ public static Optional crossrefContactEmail() { return getOptionalEnv("CROSSREF_CONTACT_EMAIL"); } + /** + * Get the API key for OpenAlex. + * + * @return the API key for OpenAlex (default unset). + */ + public static Optional openalexApiKey() { + return getOptionalEnv("OPENALEX_API_KEY"); + } + /** * Get the API credentials for GitHub. * @@ -201,4 +223,8 @@ public static Optional librariesIoKey() { public static boolean isNassaScraperEnabled() { return "true".equals(System.getenv("ENABLE_NASSA_SCRAPER")); } + + public static boolean isEscapeOssrScraperEnabled() { + return "true".equals(System.getenv("ENABLE_ESCAPE_OSSR_SCRAPER")); + } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdContributor.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdContributor.java new file mode 100644 index 0000000..fe6af22 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/RsdContributor.java @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2026 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper; + +import java.util.Optional; + +public record RsdContributor( + String givenNames, + String familyNames, + Optional affiliation, + Optional role, + Optional orcid, + int position +) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Throttler.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Throttler.java new file mode 100644 index 0000000..0701354 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/Throttler.java @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper; + +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +/** + * A thread safe throttler which will never allow more than N invocations per X time units. + * It only accounts for the start of an action, not the duration of the action. + * Actions that go over the limit are not rejected but delayed and executed when possible. + */ +public class Throttler { + + private final Semaphore semaphore; + private final long xTimeUnits; + private final TimeUnit timeUnit; + private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + runnable -> { + ThreadFactory defaultFactory = Executors.defaultThreadFactory(); + Thread t = defaultFactory.newThread(runnable); + t.setDaemon(true); + return t; + } + ); + + /** + * Create a new Throttler that will allow N invocations per X time units, in accordance with the supplied arguments. + * + * @param amountPerXTimeUnits the amount of invocations per X time units + * @param xTimeUnits how many time units can contain N invocations + * @param timeUnit the time unit of this throttler + */ + public Throttler(int amountPerXTimeUnits, long xTimeUnits, TimeUnit timeUnit) { + semaphore = new Semaphore(amountPerXTimeUnits, true); + this.xTimeUnits = xTimeUnits; + this.timeUnit = timeUnit; + } + + /** + * A blocking call that will return when you can do an action according to this throttler. + * It is up to the caller to take the action afterwards as soon as possible. + * + * @throws InterruptedException when interrupted while waiting + */ + public void awaitPermission() throws InterruptedException { + semaphore.acquire(); + releaseFuturePermit(); + } + + /** + * A non-blocking call that will run the submitted action as soon as this throttler allows for it. + * + * @param task the action that should be throttled + * @param executorService the thread pool in which the action should run + * @param the return type of the action + * @return the Future representing the running action + */ + public Future runThrottledTask(Callable task, ExecutorService executorService) { + Callable wrappedTask = () -> { + semaphore.acquire(); + releaseFuturePermit(); + return task.call(); + }; + + return executorService.submit(wrappedTask); + } + + private void releaseFuturePermit() { + scheduledExecutorService.schedule(() -> semaphore.release(), xTimeUnits, timeUnit); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/MainAggregator.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/MainAggregator.java index 65735f2..b7a256e 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/MainAggregator.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/MainAggregator.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 @@ -8,12 +8,6 @@ import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import nl.esciencecenter.rsd.scraper.Config; -import nl.esciencecenter.rsd.scraper.RsdResponseException; -import nl.esciencecenter.rsd.scraper.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.URI; import java.time.Duration; @@ -32,6 +26,11 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class MainAggregator { @@ -40,7 +39,6 @@ public class MainAggregator { static final String REMOTE_SOFTWARE_TABLE_NAME = "remote_software"; static final String AGGREGATOR_SERVICE_NAME = "Aggregator"; - public static void main(String[] args) { LOGGER.info("Start aggregating RSDs"); long start = System.nanoTime(); @@ -48,16 +46,23 @@ public static void main(String[] args) { ZonedDateTime now = ZonedDateTime.now(); Collection allRemoteEntries = PostgrestConnector.allActiveDomains(BASE_URL); - Collection remoteEntriesToScrape = allRemoteEntries.stream() - .filter(entry -> entry.refreshedAt() == null || entry.refreshedAt() - .plus(entry.refreshInterval()) - // subtracting 10 seconds to take into account variations in when this scraper starts - .minus(Duration.ofSeconds(10L)) - .isBefore(now)) + Collection remoteEntriesToScrape = allRemoteEntries + .stream() + .filter( + entry -> + entry.refreshedAt() == null || + entry + .refreshedAt() + .plus(entry.refreshInterval()) + // subtracting 10 seconds to take into account variations in when this scraper starts + .minus(Duration.ofSeconds(10L)) + .isBefore(now) + ) .toList(); ConcurrentMap softwarePerId = new ConcurrentHashMap<>(remoteEntriesToScrape.size()); - Collection> tasks = remoteEntriesToScrape.stream() + Collection> tasks = remoteEntriesToScrape + .stream() .>map(entry -> () -> { JsonArray scrapedSoftware = RemoteRsdConnector.getAllSoftware(entry.domain()); softwarePerId.put(entry.id(), scrapedSoftware); @@ -96,8 +101,7 @@ public static void main(String[] args) { jsonObject.addProperty("remote_rsd_id", id.toString()); jsonObject.addProperty("remote_software_id", jsonObject.getAsJsonPrimitive("id").getAsString()); jsonObject.remove("id"); - jsonObject - .addProperty("scraped_at", now.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + jsonObject.addProperty("scraped_at", now.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); allSoftware.add(jsonElement); } @@ -114,7 +118,9 @@ public static void main(String[] args) { JsonObject jsonObject = jsonElement.getAsJsonObject(); UUID remoteRsdId = UUID.fromString(jsonObject.getAsJsonPrimitive("remote_rsd_id").getAsString()); UUID remoteSoftwareId = UUID.fromString(jsonObject.getAsJsonPrimitive("remote_software_id").getAsString()); - Collection softwareOfRemote = softwareScrapedPerRemoteId.computeIfAbsent(remoteRsdId, id -> new HashSet<>()); + Collection softwareOfRemote = softwareScrapedPerRemoteId.computeIfAbsent(remoteRsdId, id -> + new HashSet<>() + ); softwareOfRemote.add(remoteSoftwareId); } @@ -133,13 +139,28 @@ public static void main(String[] args) { PostgrestConnector.deleteSoftware(BASE_URL, remoteId, previouslyStoredSoftwareId); } catch (RsdResponseException e) { LOGGER.error("Unknown error", e); - Utils.saveExceptionInDatabase(AGGREGATOR_SERVICE_NAME, REMOTE_SOFTWARE_TABLE_NAME, previouslyStoredSoftwareId, e); + Utils.saveExceptionInDatabase( + AGGREGATOR_SERVICE_NAME, + REMOTE_SOFTWARE_TABLE_NAME, + previouslyStoredSoftwareId, + e + ); } catch (IOException e) { LOGGER.error("Unknown error", e); - Utils.saveExceptionInDatabase(AGGREGATOR_SERVICE_NAME, REMOTE_SOFTWARE_TABLE_NAME, previouslyStoredSoftwareId, e); + Utils.saveExceptionInDatabase( + AGGREGATOR_SERVICE_NAME, + REMOTE_SOFTWARE_TABLE_NAME, + previouslyStoredSoftwareId, + e + ); } catch (InterruptedException e) { LOGGER.error("Got interrupted, early exiting deleting old entries", e); - Utils.saveExceptionInDatabase(AGGREGATOR_SERVICE_NAME, REMOTE_SOFTWARE_TABLE_NAME, previouslyStoredSoftwareId, e); + Utils.saveExceptionInDatabase( + AGGREGATOR_SERVICE_NAME, + REMOTE_SOFTWARE_TABLE_NAME, + previouslyStoredSoftwareId, + e + ); Thread.currentThread().interrupt(); return; } @@ -150,12 +171,16 @@ public static void main(String[] args) { for (RemoteRsdData entry : remoteEntriesToScrape) { UUID id = entry.id(); if (!softwarePerId.containsKey(id)) { - PostgrestConnector.updateRefreshedTimeAndErrorMessage(BASE_URL, id, now, "Unknown error while scraping"); + PostgrestConnector.updateRefreshedTimeAndErrorMessage( + BASE_URL, + id, + now, + "Unknown error while scraping" + ); } } long stop = System.nanoTime(); LOGGER.info("Done aggregating RSDs ({} ms)", (stop - start) / 1000_000L); } - } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/PostgrestConnector.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/PostgrestConnector.java index f068d10..b59cb0d 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/PostgrestConnector.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/PostgrestConnector.java @@ -9,11 +9,6 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.RsdResponseException; -import nl.esciencecenter.rsd.scraper.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.URI; import java.time.Duration; @@ -24,16 +19,20 @@ import java.util.HashSet; import java.util.List; import java.util.UUID; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PostgrestConnector { private static final Logger LOGGER = LoggerFactory.getLogger(PostgrestConnector.class); - private PostgrestConnector() { - } + private PostgrestConnector() {} public static Collection allActiveDomains(URI baseUrl) { - String filter = "select=id,domain,scrape_interval_minutes,scraped_at,remote_software(remote_software_id)&active=is.true"; + String filter = + "select=id,domain,scrape_interval_minutes,scraped_at,remote_software(remote_software_id)&active=is.true"; String url = baseUrl.toString() + "/remote_rsd?" + filter; String response = Utils.getAsAdmin(url); @@ -50,37 +49,40 @@ static List parseDomainsToScrapeResponse(String response) { JsonObject jsonObject = element.getAsJsonObject(); UUID id = UUID.fromString(jsonObject.getAsJsonPrimitive("id").getAsString()); URI domain = URI.create(jsonObject.getAsJsonPrimitive("domain").getAsString()); - Duration refreshInterval = Duration.ofMinutes(jsonObject.getAsJsonPrimitive("scrape_interval_minutes") - .getAsLong()); + Duration refreshInterval = Duration.ofMinutes( + jsonObject.getAsJsonPrimitive("scrape_interval_minutes").getAsLong() + ); JsonElement refreshedAtElement = jsonObject.get("scraped_at"); - ZonedDateTime refreshedAt = refreshedAtElement.isJsonNull() ? null : ZonedDateTime.parse(refreshedAtElement.getAsString()); + ZonedDateTime refreshedAt = refreshedAtElement.isJsonNull() + ? null + : ZonedDateTime.parse(refreshedAtElement.getAsString()); Collection softwareIds = new HashSet<>(); JsonArray idsArray = jsonObject.getAsJsonArray("remote_software"); for (JsonElement jsonElement : idsArray) { - UUID softwareId = UUID.fromString(jsonElement.getAsJsonObject() - .getAsJsonPrimitive("remote_software_id") - .getAsString()); + UUID softwareId = UUID.fromString( + jsonElement.getAsJsonObject().getAsJsonPrimitive("remote_software_id").getAsString() + ); softwareIds.add(softwareId); } - result.add(new RemoteRsdData( - id, - domain, - refreshInterval, - refreshedAt, - softwareIds - )); + result.add(new RemoteRsdData(id, domain, refreshInterval, refreshedAt, softwareIds)); } catch (RuntimeException e) { LOGGER.error("Exception when parsing item", e); - Utils.saveExceptionInDatabase(MainAggregator.AGGREGATOR_SERVICE_NAME, MainAggregator.REMOTE_SOFTWARE_TABLE_NAME, null, e); + Utils.saveExceptionInDatabase( + MainAggregator.AGGREGATOR_SERVICE_NAME, + MainAggregator.REMOTE_SOFTWARE_TABLE_NAME, + null, + e + ); } } return result; } - public static void deleteSoftware(URI baseUrl, UUID remoteRsdId, UUID remoteSoftwareId) throws RsdResponseException, IOException, InterruptedException { + public static void deleteSoftware(URI baseUrl, UUID remoteRsdId, UUID remoteSoftwareId) + throws RsdResponseException, IOException, InterruptedException { String filter = "remote_rsd_id=eq." + remoteRsdId + "&remote_software_id=eq." + remoteSoftwareId; String url = baseUrl.toString() + "/remote_software?" + filter; @@ -93,7 +95,12 @@ public static void saveRemoteSoftware(URI baseUrl, JsonArray softwareArray) { Utils.postAsAdmin(url, softwareArray.toString(), "Prefer", "resolution=merge-duplicates"); } - public static void updateRefreshedTimeAndErrorMessage(URI baseUrl, UUID id, ZonedDateTime refreshedAt, String errorMessage) { + public static void updateRefreshedTimeAndErrorMessage( + URI baseUrl, + UUID id, + ZonedDateTime refreshedAt, + String errorMessage + ) { String filter = "id=eq." + id; String url = baseUrl.toString() + "/remote_rsd?" + filter; diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdConnector.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdConnector.java index bbdfaf9..c0de897 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdConnector.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdConnector.java @@ -7,12 +7,11 @@ import com.google.gson.JsonArray; import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.RsdResponseException; -import nl.esciencecenter.rsd.scraper.Utils; - import java.io.IOException; import java.net.URI; import java.util.StringJoiner; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; public class RemoteRsdConnector { @@ -38,10 +37,10 @@ public class RemoteRsdConnector { SELECT_LIST = selectListBuilder.toString(); } - private RemoteRsdConnector() { - } + private RemoteRsdConnector() {} - public static JsonArray getAllSoftware(URI remoteDomain) throws RsdResponseException, IOException, InterruptedException { + public static JsonArray getAllSoftware(URI remoteDomain) + throws RsdResponseException, IOException, InterruptedException { String path = "/api/v1/rpc/software_overview?select=" + SELECT_LIST; String url = remoteDomain.toString() + path; diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdData.java index fc3a1f0..e493fc8 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdData.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/aggregator/RemoteRsdData.java @@ -17,5 +17,4 @@ public record RemoteRsdData( Duration refreshInterval, ZonedDateTime refreshedAt, Collection softwareIds -) { -} +) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CitationData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CitationData.java index 3af97d4..005f5df 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CitationData.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CitationData.java @@ -12,16 +12,15 @@ * Container class for Citation information retrieved from the database. */ public record CitationData( - // UUID of this entry in the database - UUID id, + // UUID of this entry in the database + UUID id, - // DOI of this entry. - Doi doi, + // DOI of this entry. + Doi doi, - // OpenAlex ID of this entry - OpenalexId openalexId, + // OpenAlex ID of this entry + OpenalexId openalexId, - // List of known DOIs citing this entry. - Collection knownDois -) { -} + // List of known DOIs citing this entry. + Collection knownDois +) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CrossrefMention.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CrossrefMention.java index 67e0a51..92f7bcf 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CrossrefMention.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/CrossrefMention.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2026 Netherlands eScience Center // SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) // SPDX-FileCopyrightText: 2022 dv4all // @@ -10,10 +10,6 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.Config; -import nl.esciencecenter.rsd.scraper.RsdResponseException; -import nl.esciencecenter.rsd.scraper.Utils; - import java.io.IOException; import java.net.URI; import java.util.ArrayList; @@ -21,17 +17,21 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; public class CrossrefMention { - static final Map crossrefTypeMap; + private static final Map crossrefTypeMap; static { - // https://api.crossref.org/types + // https://api.crossref.org/types crossrefTypeMap = new HashMap<>(); crossrefTypeMap.put("book-section", MentionType.bookSection); crossrefTypeMap.put("monograph", MentionType.other); + crossrefTypeMap.put("report-component", MentionType.report); crossrefTypeMap.put("report", MentionType.report); crossrefTypeMap.put("peer-review", MentionType.other); crossrefTypeMap.put("book-track", MentionType.book); @@ -49,6 +49,7 @@ public class CrossrefMention { crossrefTypeMap.put("proceedings-series", MentionType.conferencePaper); crossrefTypeMap.put("report-series", MentionType.report); crossrefTypeMap.put("proceedings", MentionType.conferencePaper); + crossrefTypeMap.put("database", MentionType.dataset); crossrefTypeMap.put("standard", MentionType.other); crossrefTypeMap.put("reference-book", MentionType.book); crossrefTypeMap.put("posted-content", MentionType.other); @@ -58,7 +59,6 @@ public class CrossrefMention { crossrefTypeMap.put("dataset", MentionType.dataset); crossrefTypeMap.put("book-series", MentionType.book); crossrefTypeMap.put("edited-book", MentionType.book); - crossrefTypeMap.put("standard-series", MentionType.other); } private final Doi doi; @@ -68,7 +68,9 @@ public CrossrefMention(Doi doi) { } public ExternalMentionRecord mentionData() throws IOException, InterruptedException, RsdResponseException { - StringBuilder crossrefUrlBuilder = new StringBuilder("https://api.crossref.org/works/" + Utils.urlEncode(doi.toString())); + StringBuilder crossrefUrlBuilder = new StringBuilder( + "https://api.crossref.org/works/" + Utils.urlEncode(doi.toString()) + ); Config.crossrefContactEmail().ifPresent(email -> crossrefUrlBuilder.append("?mailto=").append(email)); String responseJson = Utils.get(crossrefUrlBuilder.toString()); JsonObject jsonTree = JsonParser.parseString(responseJson).getAsJsonObject(); @@ -101,7 +103,9 @@ public ExternalMentionRecord mentionData() throws IOException, InterruptedExcept String publisher = Utils.stringOrNull(workJson.get("publisher")); Integer publicationYear = null; try { - publicationYear = Utils.integerOrNull(workJson.getAsJsonObject("published").getAsJsonArray("date-parts").get(0).getAsJsonArray().get(0)); + publicationYear = Utils.integerOrNull( + workJson.getAsJsonObject("published").getAsJsonArray("date-parts").get(0).getAsJsonArray().get(0) + ); } catch (RuntimeException e) { // year not found, we leave it at null, nothing to do } @@ -115,22 +119,25 @@ public ExternalMentionRecord mentionData() throws IOException, InterruptedExcept journal = journalBuilder.toString(); } String page = Utils.stringOrNull(workJson.get("page")); - MentionType mentionType = crossrefTypeMap.getOrDefault(Utils.stringOrNull(workJson.get("type")), MentionType.other); + MentionType mentionType = crossrefTypeMap.getOrDefault( + Utils.stringOrNull(workJson.get("type")), + MentionType.other + ); return new ExternalMentionRecord( - this.doi, - null, - null, - mentionUrl, - title, - authors, - publisher, - publicationYear, - journal, - page, - mentionType, - "Crossref", - null + this.doi, + null, + null, + mentionUrl, + title, + authors, + publisher, + publicationYear, + journal, + page, + mentionType, + "Crossref", + null ); } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataCiteReleaseRepository.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataCiteReleaseRepository.java deleted file mode 100644 index f210572..0000000 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataCiteReleaseRepository.java +++ /dev/null @@ -1,95 +0,0 @@ -// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center -// -// SPDX-License-Identifier: Apache-2.0 - -package nl.esciencecenter.rsd.scraper.doi; - -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -public class DataCiteReleaseRepository { - - private static final Logger LOGGER = LoggerFactory.getLogger(DataCiteReleaseRepository.class); - - // editorconfig-checker-disable - private static final String QUERY_UNFORMATTED = """ - query { - works(ids: [%s], first: 10000) { - nodes { - doi - versionOfCount - relatedIdentifiers { - relationType - relatedIdentifierType - relatedIdentifier - } - } - } - } - """; - // editorconfig-checker-enable - - public Map> getVersionedDois(Collection conceptDois) { - if (conceptDois.isEmpty()) { - return Collections.emptyMap(); - } - - String query = QUERY_UNFORMATTED.formatted(DataciteMentionRepository.joinDoisForGraphqlQuery(conceptDois)); - JsonObject body = new JsonObject(); - body.addProperty("query", query); - String responseJson = Utils.post("https://api.datacite.org/graphql", body.toString(), "Content-Type", "application/json"); - return parseJson(responseJson); - } - - Map> parseJson(String json) { - DataciteMentionRepository dataciteMentionRepository = new DataciteMentionRepository(); - - JsonObject root = JsonParser.parseString(json).getAsJsonObject(); - JsonArray worksJson = root.getAsJsonObject("data").getAsJsonObject("works").getAsJsonArray("nodes"); - Map> releasesPerConceptDoi = new HashMap<>(); - for (JsonElement work : worksJson) { - try { - JsonObject workObject = work.getAsJsonObject(); - String conceptDoiString = workObject.getAsJsonPrimitive("doi").getAsString(); - Doi conceptDoi = Doi.fromString(conceptDoiString); - Integer versionOfCount = Utils.integerOrNull(workObject.get("versionOfCount")); - if (versionOfCount == null || versionOfCount.intValue() != 0) { - LOGGER.debug("{} is not a concept DOI", conceptDoiString); - continue; - } - - Collection versionDois = new ArrayList<>(); - JsonArray relatedIdentifiers = workObject.getAsJsonArray("relatedIdentifiers"); - for (JsonElement relatedIdentifier : relatedIdentifiers) { - JsonObject relatedIdentifierObject = relatedIdentifier.getAsJsonObject(); - String relationType = relatedIdentifierObject.getAsJsonPrimitive("relationType").getAsString(); - if (relationType == null || !relationType.equals("HasVersion")) continue; - - String relatedIdentifierType = relatedIdentifierObject.getAsJsonPrimitive("relatedIdentifierType").getAsString(); - if (relatedIdentifierType == null || !relatedIdentifierType.equals("DOI")) continue; - - String relatedIdentifierDoi = relatedIdentifierObject.getAsJsonPrimitive("relatedIdentifier").getAsString(); - versionDois.add(Doi.fromString(relatedIdentifierDoi)); - } - Collection versionedMentions = dataciteMentionRepository.mentionData(versionDois); - - releasesPerConceptDoi.put(conceptDoi, versionedMentions); - } catch (RuntimeException e) { - LOGGER.error("Failed to scrape a DataCite mention with data {}, ", work, e); - } - } - return releasesPerConceptDoi; - } -} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteConnector.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteConnector.java new file mode 100644 index 0000000..9bd5749 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteConnector.java @@ -0,0 +1,330 @@ +// SPDX-FileCopyrightText: 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2026 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpResponse; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Throttler; +import nl.esciencecenter.rsd.scraper.Utils; + +public class DataciteConnector { + + private final String contactEmail; + private final Throttler requestThrottler; + + private static final String DOI_URL_UNFORMATTED = "https://api.datacite.org/dois/%s"; + + /** + * Constructs a connector that sends "Unidentified" requests to the DataCite REST API. + *
+ *
+ * See the DataCite API Rate Limits page for more + * information on rate limiting and authentication. + */ + public DataciteConnector() { + contactEmail = null; + requestThrottler = new Throttler(475, 5, TimeUnit.MINUTES); + } + + /** + * Constructs a connector that sends "Identified" requests to the DataCite REST API, using the supplied email address. + *
+ *
+ * See the DataCite API Rate Limits page for more + * information on rate limiting and authentication. + * + * @param contactEmail the email address to use when sending requests to the DataCite REST API + */ + public DataciteConnector(String contactEmail) { + this.contactEmail = Objects.requireNonNull(contactEmail); + requestThrottler = new Throttler(950, 5, TimeUnit.MINUTES); + } + + /** + * Harvest a mention from its DOI from DataCite. + *
+ *
+ * An example of a DataCite DOI is 10.5281/zenodo.22710673. + * Its API data can be found in + * + * this REST endpoint. + * + * + * @param doi The DOI that should be harvested. This DOI should be registered at DataCite. + * @return the harvested mention + */ + public ExternalMentionRecord harvestMention(Doi doi) + throws InterruptedException, IOException, RsdResponseException { + Objects.requireNonNull(doi); + + HttpResponse response = doiHttpRequest(doi); + if (response.statusCode() != 200) { + throw new RsdResponseException( + response.statusCode(), + response.uri(), + response.body(), + "Could not harvest DataCite mention with DOI %s".formatted(doi) + ); + } + + return DataciteParser.parseMention(response.body()); + } + + /** + * Harvest the version DOIs of a (concept) DOI. See e.g. + * + * this Zenodo explanation. + * + *
+ *
+ * An example of a concept DOI with versions is 10.5281/zenodo.6379973. + * Its versions can be found in + * + * this REST endpoint + * + * under the versions key. + *
+ *
+ * This example also shows that a concept DOI can be its own version. This method removes the concept DOI from the returned version DOIs. + * + * @param conceptDoi The (concept) DOI that should have their versions harvested. This DOI should be registered at DataCite. See the link above for an explanation of concept DOIs. + * This DOI should not be null. + * @return the DOIs that are versions of the queried DOI + */ + public Collection harvestVersionedDois(Doi conceptDoi) + throws InterruptedException, IOException, RsdResponseException { + Objects.requireNonNull(conceptDoi); + + HttpResponse response = doiHttpRequest(conceptDoi); + if (response.statusCode() != 200) { + throw new RsdResponseException( + response.statusCode(), + response.uri(), + response.body(), + "Could not harvest version DOIs of %s".formatted(conceptDoi) + ); + } + + Collection result = DataciteParser.parseVersionDois(response.body()); + result.removeIf(versionDoi -> versionDoi.equals(conceptDoi)); + return result; + } + + private HttpResponse doiHttpRequest(Doi doi) throws InterruptedException, IOException { + URI url = URI.create(DOI_URL_UNFORMATTED.formatted(doi.toUrlEncodedString())); + + this.requestThrottler.awaitPermission(); + return contactEmail == null + ? Utils.getAsHttpResponse(url) + : Utils.getAsHttpResponse(url, "User-Agent", contactEmail); + } + + static class DataciteParser { + + private static final Map dataciteTypeMap; + private static final Map dataciteTextTypeMap; + private static final Pattern URL_TREE_TAG_PATTERN = Pattern.compile("/tree/([^/]+)$"); + + static { + // https://schema.datacite.org/meta/kernel-4.7/ + dataciteTypeMap = new HashMap<>(); + dataciteTypeMap.put("Audiovisual", MentionType.presentation); + dataciteTypeMap.put("Award", MentionType.other); + dataciteTypeMap.put("Book", MentionType.book); + dataciteTypeMap.put("BookChapter", MentionType.bookSection); + dataciteTypeMap.put("Collection", MentionType.other); + dataciteTypeMap.put("ComputationalNotebook", MentionType.computerProgram); + dataciteTypeMap.put("ConferencePaper", MentionType.conferencePaper); + dataciteTypeMap.put("ConferenceProceeding", MentionType.conferencePaper); + dataciteTypeMap.put("DataPaper", MentionType.other); + dataciteTypeMap.put("Dataset", MentionType.dataset); + dataciteTypeMap.put("Dissertation", MentionType.thesis); + dataciteTypeMap.put("Event", MentionType.workshop); + dataciteTypeMap.put("Image", MentionType.other); + dataciteTypeMap.put("Instrument", MentionType.other); + dataciteTypeMap.put("InteractiveResource", MentionType.other); + dataciteTypeMap.put("Journal", MentionType.journalArticle); + dataciteTypeMap.put("JournalArticle", MentionType.journalArticle); + dataciteTypeMap.put("Model", MentionType.other); + dataciteTypeMap.put("OutputManagementPlan", MentionType.other); + dataciteTypeMap.put("PeerReview", MentionType.other); + dataciteTypeMap.put("Poster", MentionType.poster); + dataciteTypeMap.put("Preprint", MentionType.other); + dataciteTypeMap.put("Presentation", MentionType.presentation); + dataciteTypeMap.put("Project", MentionType.other); + dataciteTypeMap.put("PhysicalObject", MentionType.other); + dataciteTypeMap.put("Report", MentionType.report); + dataciteTypeMap.put("Service", MentionType.other); + dataciteTypeMap.put("Software", MentionType.computerProgram); + dataciteTypeMap.put("Sound", MentionType.other); + dataciteTypeMap.put("Standard", MentionType.other); + dataciteTypeMap.put("StudyRegistration", MentionType.thesis); + // dataciteTypeMap.put("Text", MentionType.other); + dataciteTypeMap.put("Workflow", MentionType.other); + dataciteTypeMap.put("Other", MentionType.other); + + dataciteTextTypeMap = new HashMap<>(); + dataciteTextTypeMap.put("Conference paper", MentionType.conferencePaper); + dataciteTextTypeMap.put("Dissertation", MentionType.thesis); + dataciteTextTypeMap.put("Journal article", MentionType.journalArticle); + dataciteTextTypeMap.put("Poster", MentionType.poster); + dataciteTextTypeMap.put("Presentation", MentionType.presentation); + dataciteTextTypeMap.put("Report", MentionType.report); + } + + private DataciteParser() {} + + static ExternalMentionRecord parseMention(String json) { + JsonObject root = JsonParser.parseString(json).getAsJsonObject(); + JsonObject data = root.getAsJsonObject("data"); + JsonObject attributes = data.getAsJsonObject("attributes"); + + Doi doi = Doi.fromString(attributes.getAsJsonPrimitive("doi").getAsString()); + + String rawDoiRegistrationDate = Utils.stringOrNull(attributes.get("registered")); + ZonedDateTime doiRegistrationDate = + rawDoiRegistrationDate == null ? null : ZonedDateTime.parse(rawDoiRegistrationDate); + URI url = URI.create("https://doi.org/" + Utils.urlEncode(doi.toString())); + String title = attributes + .getAsJsonArray("titles") + .get(0) + .getAsJsonObject() + .getAsJsonPrimitive("title") + .getAsString(); + + Collection authorsBuilder = new ArrayList<>(); + for (JsonElement creator : attributes.getAsJsonArray("creators")) { + addAuthor(authorsBuilder, creator.getAsJsonObject()); + } + for (JsonElement contributor : attributes.getAsJsonArray("contributors")) { + addAuthor(authorsBuilder, contributor.getAsJsonObject()); + } + String authors = String.join(", ", authorsBuilder); + + String publisher = Utils.stringOrNull(attributes.get("publisher")); + Integer publicationYear = Utils.integerOrNull(attributes.get("publicationYear")); + + MentionType mentionType; + String dataciteResourceTypeGeneral = Utils.stringOrNull( + attributes.getAsJsonObject("types").get("resourceTypeGeneral") + ); + if (dataciteResourceTypeGeneral != null && dataciteResourceTypeGeneral.equals("Text")) { + String dataciteResourceType = Utils.stringOrNull( + attributes.getAsJsonObject("types").get("resourceType") + ); + if (dataciteResourceType != null) { + dataciteResourceType = dataciteResourceType.strip(); + } + mentionType = dataciteTextTypeMap.getOrDefault(dataciteResourceType, MentionType.other); + } else { + mentionType = dataciteTypeMap.getOrDefault(dataciteResourceTypeGeneral, MentionType.other); + } + + String version = parseVersion(attributes); + + return new ExternalMentionRecord( + doi, + doiRegistrationDate, + null, + url, + title, + authors, + publisher, + publicationYear, + null, + null, + mentionType, + "DataCite", + version + ); + } + + private static String parseVersion(JsonObject attributes) { + String version = Utils.stringOrNull(attributes.get("version")); + if (version != null) { + return version; + } + + // if the version is null, we can often get the version from a linked Git URL which ends in "/tree/{tag}" + JsonArray relatedIdentifiers = attributes.getAsJsonArray("relatedIdentifiers"); + for (JsonElement relatedIdentifier : relatedIdentifiers) { + String relatedIdentifierString = Utils.stringOrNull( + relatedIdentifier.getAsJsonObject().get("relatedIdentifier") + ); + String relatedIdentifierType = Utils.stringOrNull( + relatedIdentifier.getAsJsonObject().get("relatedIdentifierType") + ); + if ( + relatedIdentifierString != null && + relatedIdentifierType != null && + relatedIdentifierType.equals("URL") + ) { + Matcher tagMatcher = URL_TREE_TAG_PATTERN.matcher(relatedIdentifierString); + if (tagMatcher.find()) { + version = tagMatcher.group(1); + break; + } + } + } + return version; + } + + static void addAuthor(Collection authors, JsonObject author) { + String givenName = Utils.stringOrNull(author.get("givenName")); + String familyName = Utils.stringOrNull(author.get("familyName")); + if (givenName == null && familyName == null) { + return; + } + + if (givenName == null) { + authors.add(familyName); + } else if (familyName == null) { + authors.add(givenName); + } else { + authors.add(givenName + " " + familyName); + } + } + + static Collection parseVersionDois(String json) { + JsonObject root = JsonParser.parseString(json).getAsJsonObject(); + JsonArray doiJsonArray = root + .getAsJsonObject("data") + .getAsJsonObject("relationships") + .getAsJsonObject("versions") + .getAsJsonArray("data"); + + Collection result = new ArrayList<>(doiJsonArray.size()); + for (JsonElement element : doiJsonArray) { + JsonObject dataObject = element.getAsJsonObject(); + if (!"dois".equals(dataObject.getAsJsonPrimitive("type").getAsString())) { + continue; + } + + String rawDoi = dataObject.getAsJsonPrimitive("id").getAsString(); + if (Doi.isValid(rawDoi)) { + result.add(Doi.fromString(rawDoi)); + } + } + + return result; + } + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java deleted file mode 100644 index d4d048a..0000000 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java +++ /dev/null @@ -1,229 +0,0 @@ -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center -// -// SPDX-License-Identifier: Apache-2.0 - -package nl.esciencecenter.rsd.scraper.doi; - -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.URI; -import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -public class DataciteMentionRepository { - - private static final Logger LOGGER = LoggerFactory.getLogger(DataciteMentionRepository.class); - - // editorconfig-checker-disable - private static final String QUERY_UNFORMATTED = """ - query { - works(ids: [%s], first: 10000) { - nodes { - doi - types { - resourceType - resourceTypeGeneral - } - version - relatedIdentifiers { - relatedIdentifier - relatedIdentifierType - } - titles(first: 1) { - title - } - publisher { - name - } - publicationYear - registered - creators { - givenName - familyName - } - contributors { - givenName - familyName - } - } - } - }"""; - // editorconfig-checker-enable - - private static final Map dataciteTypeMap; - private static final Map dataciteTextTypeMap; - private static final Pattern URL_TREE_TAG_PATTERN = Pattern.compile("/tree/([^/]+)$"); - - static { - // https://schema.datacite.org/meta/kernel-4.4/ - dataciteTypeMap = new HashMap<>(); - dataciteTypeMap.put("Audiovisual", MentionType.presentation); - dataciteTypeMap.put("Book", MentionType.book); - dataciteTypeMap.put("BookChapter", MentionType.bookSection); - dataciteTypeMap.put("Collection", MentionType.other); - dataciteTypeMap.put("ComputationalNotebook", MentionType.computerProgram); - dataciteTypeMap.put("ConferencePaper", MentionType.conferencePaper); - dataciteTypeMap.put("ConferenceProceeding", MentionType.conferencePaper); - dataciteTypeMap.put("DataPaper", MentionType.other); - dataciteTypeMap.put("Dataset", MentionType.dataset); - dataciteTypeMap.put("Dissertation", MentionType.thesis); - dataciteTypeMap.put("Event", MentionType.workshop); - dataciteTypeMap.put("Image", MentionType.other); - dataciteTypeMap.put("InteractiveResource", MentionType.other); - dataciteTypeMap.put("Journal", MentionType.journalArticle); - dataciteTypeMap.put("JournalArticle", MentionType.journalArticle); - dataciteTypeMap.put("Model", MentionType.other); - dataciteTypeMap.put("OutputManagementPlan", MentionType.other); - dataciteTypeMap.put("PeerReview", MentionType.other); - dataciteTypeMap.put("Preprint", MentionType.other); - dataciteTypeMap.put("PhysicalObject", MentionType.other); - dataciteTypeMap.put("Report", MentionType.report); - dataciteTypeMap.put("Service", MentionType.other); - dataciteTypeMap.put("Software", MentionType.computerProgram); - dataciteTypeMap.put("Sound", MentionType.other); - dataciteTypeMap.put("Standard", MentionType.other); - // dataciteTypeMap.put("Text", MentionType.other); - dataciteTypeMap.put("Workflow", MentionType.other); - dataciteTypeMap.put("Other", MentionType.other); - - dataciteTextTypeMap = new HashMap<>(); - dataciteTextTypeMap.put("Conference paper", MentionType.conferencePaper); - dataciteTextTypeMap.put("Dissertation", MentionType.thesis); - dataciteTextTypeMap.put("Journal article", MentionType.journalArticle); - dataciteTextTypeMap.put("Poster", MentionType.poster); - dataciteTextTypeMap.put("Presentation", MentionType.presentation); - dataciteTextTypeMap.put("Report", MentionType.report); - } - - // "10.5281/zenodo.1408128","10.1186/s12859-018-2165-7" - static String joinDoisForGraphqlQuery(Collection dois) { - return dois.stream() - .map(Doi::toString) - .collect(Collectors.joining("\",\"", "\"", "\"")); - } - - static Collection jsonStringToUniqueMentions(String json) { - JsonObject root = JsonParser.parseString(json).getAsJsonObject(); - JsonArray worksJson = root.getAsJsonObject("data").getAsJsonObject("works").getAsJsonArray("nodes"); - Collection mentions = new ArrayList<>(); - Set usedDois = new HashSet<>(); - for (JsonElement work : worksJson) { - try { - // Sometimes, DataCite gives back two of the same results for one DOI, e.g. for 10.4122/1.1000000817, - // so we need to only add it once, otherwise we cannot POST it to the backend - ExternalMentionRecord parsedMention = parseWork(work.getAsJsonObject()); - if (usedDois.contains(parsedMention.doi())) continue; - - usedDois.add(parsedMention.doi()); - mentions.add(parsedMention); - } catch (RuntimeException e) { - // TODO: fix exception type - LOGGER.error("Failed to scrape a DataCite mention with data {} ", work, e); - } - } - return mentions; - } - - static ExternalMentionRecord parseWork(JsonObject work) { - Doi doi = Doi.fromString(work.get("doi").getAsString()); - URI url = URI.create("https://doi.org/" + Utils.urlEncode(doi.toString())); - String title = work.getAsJsonArray("titles").get(0).getAsJsonObject().get("title").getAsString(); - - Collection authorsBuilder = new ArrayList<>(); - Iterable creators = (Iterable) work.getAsJsonArray("creators"); - for (JsonObject creator : creators) { - addAuthor(authorsBuilder, creator); - } - Iterable contributors = (Iterable) work.getAsJsonArray("contributors"); - for (JsonObject contributor : contributors) { - addAuthor(authorsBuilder, contributor); - } - String authors = String.join(", ", authorsBuilder); - - String publisher = Utils.stringOrNull(work.getAsJsonObject("publisher").get("name")); - Integer publicationYear = Utils.integerOrNull(work.get("publicationYear")); - String doiRegistrationDateString = Utils.stringOrNull(work.get("registered")); - ZonedDateTime doiRegistrationDate = null; - if (doiRegistrationDateString != null) { - doiRegistrationDate = ZonedDateTime.parse(doiRegistrationDateString); - } - - MentionType mentionType; - String dataciteResourceTypeGeneral = Utils.stringOrNull(work.getAsJsonObject("types").get("resourceTypeGeneral")); - if (dataciteResourceTypeGeneral != null && dataciteResourceTypeGeneral.equals("Text")) { - String dataciteResourceType = Utils.stringOrNull(work.getAsJsonObject("types").get("resourceType")); - if (dataciteResourceType != null) dataciteResourceType = dataciteResourceType.strip(); - mentionType = dataciteTextTypeMap.getOrDefault(dataciteResourceType, MentionType.other); - } else { - mentionType = dataciteTypeMap.getOrDefault(dataciteResourceTypeGeneral, MentionType.other); - } - String version = Utils.stringOrNull(work.get("version")); - // if the version is null, we can often get the version from a linked Git URL which ends in "/tree/{tag}" - if (version == null) { - JsonArray relatedIdentifiers = work.getAsJsonArray("relatedIdentifiers"); - for (JsonElement relatedIdentifier : relatedIdentifiers) { - String relatedIdentifierString = Utils.stringOrNull(relatedIdentifier.getAsJsonObject().get("relatedIdentifier")); - String relatedIdentifierType = Utils.stringOrNull(relatedIdentifier.getAsJsonObject().get("relatedIdentifierType")); - if (relatedIdentifierString != null && relatedIdentifierType != null && relatedIdentifierType.equals("URL")) { - Matcher tagMatcher = URL_TREE_TAG_PATTERN.matcher(relatedIdentifierString); - if (tagMatcher.find()) { - version = tagMatcher.group(1); - break; - } - } - } - } - - return new ExternalMentionRecord( - doi, - doiRegistrationDate, - null, - url, - title, - authors, - publisher, - publicationYear, - null, - null, - mentionType, - "DataCite", - version - ); - } - - static void addAuthor(Collection authors, JsonObject author) { - String givenName = Utils.stringOrNull(author.get("givenName")); - String familyName = Utils.stringOrNull(author.get("familyName")); - if (givenName == null && familyName == null) return; - if (givenName == null) authors.add(familyName); - else if (familyName == null) authors.add(givenName); - else authors.add(givenName + " " + familyName); - } - - public Collection mentionData(Collection dois) { - if (dois == null || dois.isEmpty()) { - return Collections.emptyList(); - } - - JsonObject body = new JsonObject(); - body.addProperty("query", QUERY_UNFORMATTED.formatted(joinDoisForGraphqlQuery(dois))); - String responseJson = Utils.post("https://api.datacite.org/graphql", body.toString(), "Content-Type", "application/json"); - return jsonStringToUniqueMentions(responseJson); - } -} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/Doi.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/Doi.java index 034603f..d51147f 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/Doi.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/Doi.java @@ -1,18 +1,17 @@ -// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.doi; -import nl.esciencecenter.rsd.scraper.Utils; - import java.util.Locale; import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.Utils; public class Doi { - private static final Pattern DOI_PATTERN = Pattern.compile("^10(\\.\\w+)+/\\S+$"); + private static final Pattern DOI_PATTERN = Pattern.compile("^10(\\.\\w++)++/\\S+$"); private final String doiString; diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ExternalMentionRecord.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ExternalMentionRecord.java index 89973a2..ebd330e 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ExternalMentionRecord.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ExternalMentionRecord.java @@ -9,18 +9,17 @@ import java.time.ZonedDateTime; public record ExternalMentionRecord( - Doi doi, - ZonedDateTime doiRegistrationDate, - OpenalexId openalexId, - URI url, - String title, - String authors, - String publisher, - Integer publicationYear, - String journal, - String page, - MentionType mentionType, - String source, - String version -) { -} + Doi doi, + ZonedDateTime doiRegistrationDate, + OpenalexId openalexId, + URI url, + String title, + String authors, + String publisher, + Integer publicationYear, + String journal, + String page, + MentionType mentionType, + String source, + String version +) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainCitations.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainCitations.java index da9559c..61086ad 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainCitations.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainCitations.java @@ -1,20 +1,21 @@ -// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2023 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.doi; -import nl.esciencecenter.rsd.scraper.Config; -import nl.esciencecenter.rsd.scraper.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; +import java.util.Optional; import java.util.UUID; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /* * Main entry point for citation scraper. @@ -24,70 +25,110 @@ public class MainCitations { private static final Logger LOGGER = LoggerFactory.getLogger(MainCitations.class); public static void main(String[] args) { - LOGGER.info("Start scraping citations"); long start = System.currentTimeMillis(); try { - // Connect to the database to retrieve the reference papers to scrape + scrapeCitations(); + } catch (RuntimeException | IOException e) { + LOGGER.error("Exception while scraping citations", e); + Utils.saveExceptionInDatabase("Citation scraper", null, null, e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOGGER.warn("Got interrupted scraping citations, exiting"); + return; + } - String backendUrl = Config.backendBaseUrl(); - PostgrestCitationRepository localCitationRepository = new PostgrestCitationRepository(backendUrl); + long time = System.currentTimeMillis() - start; + LOGGER.info("Done scraping citations ({} ms.)", time); + } - Collection referencePapersToScrape = localCitationRepository.leastRecentlyScrapedCitations(5); - OpenAlexConnector openAlexConnector = new OpenAlexConnector(); - PostgrestMentionRepository localMentionRepository = new PostgrestMentionRepository(backendUrl); - String email = Config.crossrefContactEmail().orElse(null); - Instant now = Instant.now(); + private static void scrapeCitations() throws IOException, InterruptedException { + // Connect to the database to retrieve the reference papers to scrape + String backendUrl = Config.backendBaseUrl(); + PostgrestCitationRepository localCitationRepository = new PostgrestCitationRepository(backendUrl); - for (CitationData citationData : referencePapersToScrape) { + Collection referencePapersToScrape = localCitationRepository.leastRecentlyScrapedCitations( + Config.maxCitationSourcesToScrape() + ); - long t1 = System.currentTimeMillis(); + // already update the citations_scraped_at fields to prevent them being scraped by a second process if this one takes too long + localCitationRepository.updateScrapedAtTime( + referencePapersToScrape.stream().map(CitationData::id).toList(), + Instant.now() + ); - LOGGER.info("Scraping for DOI {}, OpenAlex ID {}", citationData.doi(), citationData.openalexId()); + OpenAlexConnector openAlexConnector = new OpenAlexConnector(); + PostgrestMentionRepository localMentionRepository = new PostgrestMentionRepository(backendUrl); + Instant now = Instant.now(); - Collection citingMentions = openAlexConnector.citations(citationData.openalexId(), citationData.doi(), email, citationData.id()); - // we don't update mentions that have a DOI in the database with OpenAlex data, as they can already be - // scraped through Crossref or DataCite + for (CitationData citationData : referencePapersToScrape) { + long t1 = System.currentTimeMillis(); - long t2 = System.currentTimeMillis(); + LOGGER.info("Scraping for DOI {}, OpenAlex ID {}", citationData.doi(), citationData.openalexId()); - citingMentions.removeIf(mention -> mention.doi() != null && citationData.knownDois().contains(mention.doi())); - Collection savedIds = new ArrayList<>(citingMentions.size()); - for (ExternalMentionRecord citingMention : citingMentions) { - try { - RsdMentionIds ids = localMentionRepository.createOrUpdateMentionWithOpenalexId(citingMention, now); - savedIds.add(ids); - } catch (Exception e) { - LOGGER.error("Unable to save exception with OpenAlex ID {}", citingMention.openalexId()); - Utils.saveExceptionInDatabase("Citation scraper", "mention", null, e); + Collection citingMentions; + try { + OpenalexId openalexId = citationData.openalexId(); + if (openalexId == null) { + if (citationData.doi() == null) { + continue; } - } - Collection citingMentionIds = new ArrayList<>(); - for (RsdMentionIds ids : savedIds) { - citingMentionIds.add(ids.id()); - } + Optional optionalOpenalexId = openAlexConnector.getOpenAlexIdFromDoi( + citationData.doi() + ); + if (optionalOpenalexId.isEmpty()) { + continue; + } - long t3 = System.currentTimeMillis(); + openalexId = optionalOpenalexId.get(); + } + citingMentions = openAlexConnector.citations(openalexId, citationData.id()); + } catch (RsdResponseException e) { + LOGGER.error("Exception while scraping citations", e); + Utils.saveExceptionInDatabase("Citation scraper", null, citationData.id(), e); + continue; + } + // we don't update mentions that have a DOI in the database with OpenAlex data, as they can already be + // scraped through Crossref or DataCite + + long t2 = System.currentTimeMillis(); + + citingMentions.removeIf( + mention -> mention.doi() != null && citationData.knownDois().contains(mention.doi()) + ); + Collection savedIds = new ArrayList<>(citingMentions.size()); + for (ExternalMentionRecord citingMention : citingMentions) { + try { + RsdMentionIds ids = localMentionRepository.createOrUpdateMentionWithOpenalexId(citingMention, now); + savedIds.add(ids); + } catch (Exception e) { + LOGGER.error("Unable to save exception with OpenAlex ID {}", citingMention.openalexId()); + Utils.saveExceptionInDatabase("Citation scraper", "mention", null, e); + } + } - localCitationRepository.saveCitations(backendUrl, citationData.id(), citingMentionIds, now); + Collection citingMentionIds = new ArrayList<>(); + for (RsdMentionIds ids : savedIds) { + citingMentionIds.add(ids.id()); + } - long t4 = System.currentTimeMillis(); + long t3 = System.currentTimeMillis(); - LOGGER.info("Scraping for {} done. OpenAlex: {} ms. Saving mentions {} ms. Saving citations {} ms. Total {} ms.", citationData.doi(), (t2 - t1), (t3 - t2), (t4 - t3), (t4 - t1)); - } + localCitationRepository.saveCitations(backendUrl, citationData.id(), citingMentionIds); - } catch (IOException | InterruptedException e) { - Utils.saveExceptionInDatabase("Citation scraper", null, null, e); + long t4 = System.currentTimeMillis(); - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } + LOGGER.info( + "Scraping for {} done. OpenAlex: {} ms. Saving mentions {} ms. Saving citations {} ms. Total {} ms.", + citationData.doi(), + t2 - t1, + t3 - t2, + t4 - t3, + t4 - t1 + ); } - - long time = System.currentTimeMillis() - start; - LOGGER.info("Done scraping citations ({} ms.)", time); } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainMentions.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainMentions.java index 4661e27..4c8b7cf 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainMentions.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainMentions.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 @@ -9,34 +9,34 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.Config; -import nl.esciencecenter.rsd.scraper.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class MainMentions { private static final Logger LOGGER = LoggerFactory.getLogger(MainMentions.class); - public static void main(String[] args) { + static void main() { + LOGGER.info("Start harvesting mentions"); - LOGGER.info("Start scraping mentions"); - - long t1 = System.currentTimeMillis(); + Instant tic = Instant.now(); PostgrestMentionRepository localMentionRepository = new PostgrestMentionRepository(Config.backendBaseUrl()); - Collection mentionsToScrape = localMentionRepository.leastRecentlyScrapedMentions(Config.maxRequestsDoi()); + Collection mentionsToScrape = localMentionRepository.leastRecentlyScrapedMentions( + Config.maxRequestsDoi() + ); // we will remove successfully scraped mentions from here, // we use this to set scrapedAt even for failed mentions, // to put them back at the scraping queue @@ -58,11 +58,12 @@ public static void main(String[] args) { } } - String doisJoined = mentionsToScrape.stream() - .map(RsdMentionIds::doi) - .filter(Objects::nonNull) - .map(Doi::toUrlEncodedString) - .collect(Collectors.joining(",")); + String doisJoined = mentionsToScrape + .stream() + .map(RsdMentionIds::doi) + .filter(Objects::nonNull) + .map(Doi::toUrlEncodedString) + .collect(Collectors.joining(",")); String jsonSources = null; try { jsonSources = Utils.get("https://doi.org/doiRA/" + doisJoined); @@ -77,18 +78,32 @@ public static void main(String[] args) { // DATACITE final String dataciteDoiRaKey = "DataCite"; - Collection dataciteDois = doiToSource.entrySet() - .stream() - .filter(doiSourceEntry -> doiSourceEntry.getValue().equals(dataciteDoiRaKey)) - .map(Map.Entry::getKey) - .map(Doi::fromString) - .toList(); - Collection scrapedDataciteMentions = List.of(); - try { - scrapedDataciteMentions = new DataciteMentionRepository().mentionData(dataciteDois); - } catch (RuntimeException e) { - Exception exceptionToSave = new Exception("Failed scraping the following DataCite DOIs: " + dataciteDois, e); - Utils.saveExceptionInDatabase("DataCite mention scraper", "mention", null, exceptionToSave); + Collection dataciteDois = doiToSource + .entrySet() + .stream() + .filter(doiSourceEntry -> doiSourceEntry.getValue().equals(dataciteDoiRaKey)) + .map(Map.Entry::getKey) + .map(Doi::fromString) + .toList(); + Collection scrapedDataciteMentions = new ArrayList<>(dataciteDois.size()); + DataciteConnector dataciteConnector = Config.crossrefContactEmail() + .map(DataciteConnector::new) + .orElseGet(DataciteConnector::new); + for (Doi dataciteDoi : dataciteDois) { + try { + scrapedDataciteMentions.add(dataciteConnector.harvestMention(dataciteDoi)); + } catch (Exception e) { + Exception exceptionToSave = new Exception( + "Failed scraping the following DataCite DOI: " + dataciteDois, + e + ); + Utils.saveExceptionInDatabase("DataCite mention scraper", "mention", null, exceptionToSave); + + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + return; + } + } } for (ExternalMentionRecord scrapedMention : scrapedDataciteMentions) { Doi doi = scrapedMention.doi(); @@ -101,18 +116,18 @@ public static void main(String[] args) { LOGGER.error("Failed to update a DataCite mention with DOI {}", scrapedMention.doi()); Utils.saveExceptionInDatabase("Mention scraper", "mention", id, e); } - } // END DATACITE // CROSSREF final String crossrefDoiRaKey = "Crossref"; - Collection crossrefDois = doiToSource.entrySet() - .stream() - .filter(doiSourceEntry -> doiSourceEntry.getValue().equals(crossrefDoiRaKey)) - .map(Map.Entry::getKey) - .map(Doi::fromString) - .toList(); + Collection crossrefDois = doiToSource + .entrySet() + .stream() + .filter(doiSourceEntry -> doiSourceEntry.getValue().equals(crossrefDoiRaKey)) + .map(Map.Entry::getKey) + .map(Doi::fromString) + .toList(); for (Doi crossrefDoi : crossrefDois) { ExternalMentionRecord scrapedMention; UUID id = doiToId.get(crossrefDoi); @@ -120,7 +135,10 @@ public static void main(String[] args) { scrapedMention = new CrossrefMention(crossrefDoi).mentionData(); } catch (Exception e) { LOGGER.error("Failed to scrape a Crossref mention with DOI {}", crossrefDoi); - RuntimeException exceptionWithMessage = new RuntimeException("Failed to scrape a Crossref mention with DOI " + crossrefDoi, e); + RuntimeException exceptionWithMessage = new RuntimeException( + "Failed to scrape a Crossref mention with DOI " + crossrefDoi, + e + ); Utils.saveExceptionInDatabase("Crossref mention scraper", "mention", id, exceptionWithMessage); continue; } @@ -129,44 +147,65 @@ public static void main(String[] args) { localMentionRepository.updateMention(mentionToUpdate, false); mentionsFailedToScrape.remove(id); } catch (Exception e) { - RuntimeException exceptionWithMessage = new RuntimeException("Failed to update a Crossref mention with DOI " + crossrefDoi, e); + RuntimeException exceptionWithMessage = new RuntimeException( + "Failed to update a Crossref mention with DOI " + crossrefDoi, + e + ); Utils.saveExceptionInDatabase("Crossref mention scraper", "mention", id, exceptionWithMessage); } } // END CROSSREF // OPENALEX (other DOI registry agents) - String email = Config.crossrefContactEmail().orElse(null); Collection scrapedOpenalexMentions = new ArrayList<>(); OpenAlexConnector openAlexConnector = new OpenAlexConnector(); - Collection invalidDoiRas = Set.of(dataciteDoiRaKey, crossrefDoiRaKey, "Invalid DOI", "DOI does not exist", "Unknown"); - Collection europeanPublicationsOfficeDois = doiToSource.entrySet() - .stream() - .filter(doiSourceEntry -> !invalidDoiRas.contains(doiSourceEntry.getValue())) - .map(Map.Entry::getKey) - .map(Doi::fromString) - .toList(); + Collection invalidDoiRas = Set.of( + dataciteDoiRaKey, + crossrefDoiRaKey, + "Invalid DOI", + "DOI does not exist", + "Unknown" + ); + Collection europeanPublicationsOfficeDois = doiToSource + .entrySet() + .stream() + .filter(doiSourceEntry -> !invalidDoiRas.contains(doiSourceEntry.getValue())) + .map(Map.Entry::getKey) + .map(Doi::fromString) + .toList(); try { - scrapedOpenalexMentions.addAll(openAlexConnector.mentionDataByDois(europeanPublicationsOfficeDois, email)); + scrapedOpenalexMentions.addAll(openAlexConnector.mentionDataByDois(europeanPublicationsOfficeDois)); } catch (Exception e) { - Exception exceptionToSave = new Exception("Failed scraping the following EPO DOIs: " + europeanPublicationsOfficeDois, e); + Exception exceptionToSave = new Exception( + "Failed scraping the following EPO DOIs: " + europeanPublicationsOfficeDois, + e + ); Utils.saveExceptionInDatabase("OpenAlex mention scraper", "mention", null, exceptionToSave); } Collection openalexIdsToScrape = mentionsToScrape - .stream() - .filter(ids -> ids.doi() == null && ids.openalexId() != null) - .map(RsdMentionIds::openalexId) - .toList(); + .stream() + .filter(ids -> ids.doi() == null && ids.openalexId() != null) + .map(RsdMentionIds::openalexId) + .toList(); try { - scrapedOpenalexMentions.addAll(openAlexConnector.mentionDataByOpenalexIds(openalexIdsToScrape, email)); + scrapedOpenalexMentions.addAll(openAlexConnector.mentionDataByOpenalexIds(openalexIdsToScrape)); } catch (Exception e) { - Exception exceptionToSave = new Exception("Failed scraping the following OpenAlex IDs: " + openalexIdsToScrape, e); + Exception exceptionToSave = new Exception( + "Failed scraping the following OpenAlex IDs: " + openalexIdsToScrape, + e + ); Utils.saveExceptionInDatabase("OpenAlex mention scraper", "mention", null, exceptionToSave); } for (ExternalMentionRecord scrapedMention : scrapedOpenalexMentions) { OpenalexId openalexId = scrapedMention.openalexId(); UUID id = openalexIdToId.get(openalexId); + // this shouldn't normally happen, but the data-generation script has + // OpenAlex mentions without storing an OpenAlex ID + if (id == null) { + id = doiToId.get(scrapedMention.doi()); + } + RsdMentionRecord mentionToUpdate = new RsdMentionRecord(id, scrapedMention, now); try { localMentionRepository.updateMention(mentionToUpdate, true); @@ -187,8 +226,8 @@ public static void main(String[] args) { } } - long time = System.currentTimeMillis() - t1; - LOGGER.info("Done scraping mentions ({} ms.)", time); + Instant toc = Instant.now(); + LOGGER.info("Done harvesting mentions ({} ms.)", Duration.between(tic, toc).toMillis()); } static Map parseJsonDoiSources(String jsonSources) { diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainReleases.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainReleases.java index 98cb886..240704e 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainReleases.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MainReleases.java @@ -1,65 +1,164 @@ -// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2023 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.doi; -import nl.esciencecenter.rsd.scraper.Config; -import nl.esciencecenter.rsd.scraper.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /* - * 1. Get the least recently scraped releases from software with a concept DOI. We also check for existing releases that already exist as a mention in the database, so we don't have to (TODO) recreate them later. - * 2. For each release check if it's a concept DOI on DataCite and get all the versioned DOIs. - * 3. For each versioned DOI, get its metadata from DataCite. If this versioned DOI did not exist as a mention, create it using the mention scraper. Add each versioned DOI as an entry in the release_content table + * 1. Get the least recently scraped releases from software with a concept DOI. We also check for existing releases that already exist as a mention in the database, so we don't have to re-harvest them later. + * 2. For each release get all the versioned DOIs. + * 3. For each versioned DOI, get its metadata from DataCite. If this versioned DOI did not exist as a mention, create it using the mention harvester. Add each versioned DOI as an entry in the release_content table */ public class MainReleases { private static final Logger LOGGER = LoggerFactory.getLogger(MainReleases.class); + private static final String SERVICE_NAME = "Releases harvester"; + private static final String TABLE_NAME = "software"; - public static void main(String[] args) { + static void main() { + LOGGER.info("Start harvesting releases"); - LOGGER.info("Start scraping releases"); + Instant tic = Instant.now(); - long t1 = System.currentTimeMillis(); + try { + harvestReleases(); + } catch (RuntimeException e) { + Utils.saveExceptionInDatabase(SERVICE_NAME, TABLE_NAME, null, e); + } + + Instant toc = Instant.now(); + LOGGER.info("Done harvesting releases ({} ms.)", Duration.between(tic, toc).toMillis()); + } + private static void harvestReleases() { PostgrestReleaseRepository releaseRepository = new PostgrestReleaseRepository(Config.backendBaseUrl()); - Collection releasesToScrape = releaseRepository.leastRecentlyScrapedReleases(Config.maxRequestsDoi()); + Collection releasesToHarvest = releaseRepository.leastRecentlyScrapedReleases( + Config.maxRequestsDoi() + ); - Collection conceptDoisToScrape = releasesToScrape.stream() - .map(releaseData -> releaseData.conceptDoi) - .toList(); + Collection conceptDoisToScrape = releasesToHarvest + .stream() + .map(releaseData -> releaseData.conceptDoi) + .toList(); + Map conceptIdToDatabaseId = releasesToHarvest + .stream() + .collect(Collectors.toMap(entry -> entry.conceptDoi, entry -> entry.softwareId)); + Map> conceptIdToExistingReleases = releasesToHarvest + .stream() + .collect(Collectors.toMap(entry -> entry.conceptDoi, entry -> entry.versionedDois)); - Map> scrapedReleasesPerConceptDoi = new DataCiteReleaseRepository().getVersionedDois(conceptDoisToScrape); + DataciteConnector dataciteConnector = Config.crossrefContactEmail() + .map(DataciteConnector::new) + .orElseGet(DataciteConnector::new); - Instant now = Instant.now(); + Collection> harvestVersionDoiTasks = new ConcurrentLinkedQueue<>(); + ConcurrentMap> scrapedReleasesPerConceptDoi = new ConcurrentHashMap<>(); + try (ExecutorService threadPool = Executors.newFixedThreadPool(20)) { + for (Doi conceptDoi : conceptDoisToScrape) { + Runnable harvestVersionDoisTask = () -> { + Collection versionDois; + try { + versionDois = dataciteConnector.harvestVersionedDois(conceptDoi); + } catch (Exception e) { + Utils.saveExceptionInDatabase( + SERVICE_NAME, + TABLE_NAME, + conceptIdToDatabaseId.get(conceptDoi), + e + ); + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + return; + } + + for (Doi versionDoi : versionDois) { + if (conceptIdToExistingReleases.get(conceptDoi).contains(versionDoi)) { + continue; + } + + Runnable harvestMentionTask = () -> { + ExternalMentionRecord harvestedMention; + try { + harvestedMention = dataciteConnector.harvestMention(versionDoi); + } catch (Exception e) { + Utils.saveExceptionInDatabase( + SERVICE_NAME, + TABLE_NAME, + conceptIdToDatabaseId.get(conceptDoi), + e + ); + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + return; + } + + Collection harvestedMentions = + scrapedReleasesPerConceptDoi.computeIfAbsent(conceptDoi, _ -> + new ConcurrentLinkedQueue<>() + ); + harvestedMentions.add(harvestedMention); + }; + + threadPool.submit(harvestMentionTask); + } + }; + + harvestVersionDoiTasks.add(threadPool.submit(harvestVersionDoisTask)); + } + + for (Future future : harvestVersionDoiTasks) { + try { + future.get(); + } catch (Exception e) { + Utils.saveExceptionInDatabase(SERVICE_NAME, TABLE_NAME, null, e); + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + return; + } + } + } + } + + Instant harvestTime = Instant.now(); PostgrestMentionRepository localMentionRepository = new PostgrestMentionRepository(Config.backendBaseUrl()); - Collection allMentions = scrapedReleasesPerConceptDoi.values().stream() - .flatMap(Collection::stream) - .toList(); + Collection allMentions = scrapedReleasesPerConceptDoi + .values() + .stream() + .flatMap(Collection::stream) + .toList(); Map doiToId = new HashMap<>(); for (ExternalMentionRecord mention : allMentions) { try { - RsdMentionIds ids = localMentionRepository.createMentionIfNotExistsOnDoiAndGetIds(mention, now); + RsdMentionIds ids = localMentionRepository.createMentionIfNotExistsOnDoiAndGetIds(mention, harvestTime); doiToId.put(mention.doi(), ids.id()); } catch (Exception e) { LOGGER.error("Unable to save mention with DOI {}", mention.doi()); - Utils.saveExceptionInDatabase("Releases scraper", "mention", null, e); + Utils.saveExceptionInDatabase(SERVICE_NAME, TABLE_NAME, null, e); } } - releaseRepository.saveReleaseContent(releasesToScrape, scrapedReleasesPerConceptDoi, doiToId); - - long time = System.currentTimeMillis() - t1; - LOGGER.info("Done scraping releases ({} ms.)", time); + releaseRepository.saveReleaseContent(releasesToHarvest, scrapedReleasesPerConceptDoi, doiToId); } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MentionType.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MentionType.java index 793a965..977b4bf 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MentionType.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MentionType.java @@ -24,5 +24,5 @@ public enum MentionType { videoRecording, webpage, workshop, - other + other, } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnector.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnector.java index 7715843..7830760 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnector.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnector.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2023 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 @@ -9,53 +9,114 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.URI; import java.net.http.HttpResponse; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Throttler; +import nl.esciencecenter.rsd.scraper.Utils; class OpenAlexConnector { - private static final Logger LOGGER = LoggerFactory.getLogger(OpenAlexConnector.class); + private static final String DOI_FILTER_URL_UNFORMATTED = "https://api.openalex.org/works?filter=doi:%s"; + private static final String OPENALEX_ID_URL_UNFORMATTED = "https://api.openalex.org/works?filter=ids.openalex:%s"; + private static final String OPENALEX_CITES_URL_UNFORMATTED = "https://api.openalex.org/works?filter=cites:%s"; + // https://developers.openalex.org/api-reference/authentication#exceeding-limits + // The docs state the rate limit is 100 requests per second in addition to the pricing model. + // We make it slightly lower to be safe. + private static final Throttler apiThrottler = new Throttler(90, 1050, TimeUnit.MILLISECONDS); + + private static final Map openalexTypeMap = new HashMap<>(); + + static { + // https://developers.openalex.org/api-reference/work-types + openalexTypeMap.put("article", MentionType.journalArticle); + openalexTypeMap.put("book", MentionType.book); + openalexTypeMap.put("book-chapter", MentionType.bookSection); + openalexTypeMap.put("dataset", MentionType.dataset); + openalexTypeMap.put("dissertation", MentionType.thesis); + openalexTypeMap.put("editorial", MentionType.journalArticle); + openalexTypeMap.put("erratum", MentionType.other); + openalexTypeMap.put("letter", MentionType.other); + openalexTypeMap.put("libguides", MentionType.other); + openalexTypeMap.put("other", MentionType.other); + openalexTypeMap.put("paratext", MentionType.other); + openalexTypeMap.put("peer-review", MentionType.other); + openalexTypeMap.put("preprint", MentionType.other); + openalexTypeMap.put("reference-entry", MentionType.other); + openalexTypeMap.put("report", MentionType.report); + openalexTypeMap.put("retraction", MentionType.other); + openalexTypeMap.put("review", MentionType.other); + openalexTypeMap.put("standard", MentionType.other); + openalexTypeMap.put("supplementary-materials", MentionType.other); + } + + private static String doOpenAlexGetRequest(String url) + throws IOException, InterruptedException, RsdResponseException { + Objects.requireNonNull(url); + + String apiKey = Config.openalexApiKey().orElse(null); - static final String DOI_FILTER_URL_UNFORMATTED = "https://api.openalex.org/works?filter=doi:%s"; - static final String OPENALEX_ID_URL_UNFORMATTED = "https://api.openalex.org/works?filter=ids.openalex:%s"; + apiThrottler.awaitPermission(); + HttpResponse response = + apiKey == null || apiKey.isBlank() + ? Utils.getAsHttpResponse(url) + : Utils.getAsHttpResponse(url + "&api_key=" + apiKey); + + if (response.statusCode() == 200) { + return response.body(); + } else { + throw new RsdResponseException( + response.statusCode(), + response.uri(), + response.body(), + "Unexpected response on GET request to OpenAlex" + ); + } + } - public Collection mentionDataByDois(Collection dois, String email) throws IOException, InterruptedException { + public Optional getOpenAlexIdFromDoi(Doi doi) + throws RsdResponseException, IOException, InterruptedException { + if (doi == null) { + return Optional.empty(); + } + + Collection harvestedMention = mentionDataByDois(List.of(doi)); + if (harvestedMention.isEmpty()) { + return Optional.empty(); + } + + return Optional.of(harvestedMention.iterator().next().openalexId()); + } + + public Collection mentionDataByDois(Collection dois) + throws IOException, InterruptedException, RsdResponseException { if (dois == null || dois.isEmpty()) { return Collections.emptyList(); } - String filter = dois - .stream() - .filter(Objects::nonNull) - .map(Doi::toString) - .collect(Collectors.joining("|")); + String filter = dois.stream().filter(Objects::nonNull).map(Doi::toString).collect(Collectors.joining("|")); // e.g. https://api.openalex.org/works?filter=doi:10.1038%2Fs41598-024-73248-4|10.5194%2Ftc-2022-249-rc1&per-page=200 String worksUri = DOI_FILTER_URL_UNFORMATTED.formatted(Utils.urlEncode(filter)) + "&per-page=200"; - HttpResponse response; - if (email == null || email.isBlank()) { - response = Utils.getAsHttpResponse(worksUri); - } else { - response = Utils.getAsHttpResponse(worksUri, "User-Agent", "mailto:" + email); - } + String responseBody = doOpenAlexGetRequest(worksUri); - JsonObject tree = JsonParser.parseString(response.body()).getAsJsonObject(); - JsonArray mentionsArray = tree - .getAsJsonArray("results"); + JsonObject tree = JsonParser.parseString(responseBody).getAsJsonObject(); + JsonArray mentionsArray = tree.getAsJsonArray("results"); Collection mentions = new ArrayList<>(); for (JsonElement mention : mentionsArray) { @@ -72,29 +133,24 @@ public Collection mentionDataByDois(Collection dois, return mentions; } - public Collection mentionDataByOpenalexIds(Collection openalexIds, String email) throws IOException, InterruptedException { + public Collection mentionDataByOpenalexIds(Collection openalexIds) + throws IOException, InterruptedException, RsdResponseException { if (openalexIds == null || openalexIds.isEmpty()) { return Collections.emptyList(); } String filter = openalexIds - .stream() - .filter(Objects::nonNull) - .map(OpenalexId::getOpenalexKey) - .collect(Collectors.joining("|")); + .stream() + .filter(Objects::nonNull) + .map(OpenalexId::getOpenalexKey) + .collect(Collectors.joining("|")); // e.g. https://api.openalex.org/works?filter=ids.openalex:W4402994101|W4319593220&per-page=200 String worksUri = OPENALEX_ID_URL_UNFORMATTED.formatted(Utils.urlEncode(filter)) + "&per-page=200"; - HttpResponse response; - if (email == null || email.isBlank()) { - response = Utils.getAsHttpResponse(worksUri); - } else { - response = Utils.getAsHttpResponse(worksUri, "User-Agent", "mailto:" + email); - } + String responseBody = doOpenAlexGetRequest(worksUri); - JsonObject tree = JsonParser.parseString(response.body()).getAsJsonObject(); - JsonArray mentionsArray = tree - .getAsJsonArray("results"); + JsonObject tree = JsonParser.parseString(responseBody).getAsJsonObject(); + JsonArray mentionsArray = tree.getAsJsonArray("results"); Collection mentions = new ArrayList<>(); for (JsonElement mention : mentionsArray) { @@ -111,88 +167,40 @@ public Collection mentionDataByOpenalexIds(Collection citations(OpenalexId openalexId, Doi doi, String email, UUID id) throws IOException, InterruptedException { + public Collection citations(OpenalexId openalexId, UUID id) + throws IOException, InterruptedException, RsdResponseException { // This shouldn't happen, but let's check it to prevent unexpected exceptions: - if (doi == null && openalexId == null) { - return Collections.emptyList(); - } - - String worksUri = openalexId != null - ? OPENALEX_ID_URL_UNFORMATTED.formatted(openalexId.toUrlEncodedString()) - : DOI_FILTER_URL_UNFORMATTED.formatted(doi.toUrlEncodedString()); - - Optional optionalCitationsUri = citationsUri(worksUri, email); - if (optionalCitationsUri.isEmpty()) { + if (openalexId == null) { return Collections.emptyList(); } - return scrapeCitations(optionalCitationsUri.get(), email, id); + String citationsUri = citationsUri(openalexId); + return scrapeCitations(citationsUri, id); } - static Optional citationsUri(String worksUri, String email) throws IOException, InterruptedException { - HttpResponse response; - if (email == null || email.isBlank()) { - response = Utils.getAsHttpResponse(worksUri); - } else { - response = Utils.getAsHttpResponse(worksUri, "User-Agent", "mailto:" + email); - } - - JsonObject tree = JsonParser.parseString(response.body()).getAsJsonObject(); - - int count = tree - .getAsJsonObject("meta") - .getAsJsonPrimitive("count") - .getAsInt(); - - if (count < 1) { - LOGGER.warn("No results found for {}: {}", worksUri, count); - return Optional.empty(); - } - - if (count > 1) { - LOGGER.warn("More than 1 result found for {}: {}, taking the first", worksUri, count); - } - - String citationsUri = null; - try { - citationsUri = tree - .getAsJsonArray("results") - .get(0) - .getAsJsonObject() - .getAsJsonPrimitive("cited_by_api_url") - .getAsString(); - } catch (RuntimeException e) { - LOGGER.error("Exception parsing cited_by_api_url for %s".formatted(worksUri), e); - Utils.saveExceptionInDatabase("OpenAlex citations scraper", null, null, e); - } - - return Optional.ofNullable(citationsUri); + // https://docs.openalex.org/api-entities/works/filter-works#cites + static String citationsUri(OpenalexId openalexId) { + Objects.requireNonNull(openalexId); + return OPENALEX_CITES_URL_UNFORMATTED.formatted(openalexId.getOpenalexKey()); } // we use cursor paging as that will always work // https://docs.openalex.org/how-to-use-the-api/get-lists-of-entities/paging#cursor-paging - static Collection scrapeCitations(String citationsUri, String email, UUID id) throws IOException, InterruptedException { + // the id parameter is used only for logging exceptions + static Collection scrapeCitations(String citationsUri, UUID id) + throws IOException, InterruptedException, RsdResponseException { final int perPage = 200; String cursor = "*"; Collection citations = new ArrayList<>(); while (cursor != null) { - HttpResponse response; String citationsUriWithCursor = citationsUri + "&per-page=" + perPage + "&cursor=" + cursor; - if (email == null || email.isBlank()) { - response = Utils.getAsHttpResponse(citationsUriWithCursor); - } else { - response = Utils.getAsHttpResponse(citationsUriWithCursor, "User-Agent", "mailto:" + email); - } - JsonObject tree = JsonParser.parseString(response.body()).getAsJsonObject(); + String responseBody = doOpenAlexGetRequest(citationsUriWithCursor); + JsonObject tree = JsonParser.parseString(responseBody).getAsJsonObject(); - cursor = Utils.stringOrNull(tree - .getAsJsonObject("meta") - .get("next_cursor") - ); + cursor = Utils.stringOrNull(tree.getAsJsonObject("meta").get("next_cursor")); - JsonArray citationsArray = tree - .getAsJsonArray("results"); + JsonArray citationsArray = tree.getAsJsonArray("results"); for (JsonElement citation : citationsArray) { ExternalMentionRecord citationAsMention; @@ -230,46 +238,46 @@ static ExternalMentionRecord parseAsMention(JsonElement element) { String title = Utils.stringOrNull(citationObject.get("title")); if (title == null) { String openAlexId = citationObject.getAsJsonPrimitive("id").getAsString(); - String message = "The title of the mention with DOI %s and OpenAlex ID %s is null".formatted(doiString, openAlexId); + String message = "The title of the mention with DOI %s and OpenAlex ID %s is null".formatted( + doiString, + openAlexId + ); throw new RuntimeException(message); } JsonArray authorsArray = citationObject.getAsJsonArray("authorships"); String authors = StreamSupport.stream(authorsArray.spliterator(), false) - .map(JsonElement::getAsJsonObject) - .map(jo -> jo.get("raw_author_name")) - .filter(Predicate.not(JsonElement::isJsonNull)) - .map(JsonElement::getAsString) - .collect(Collectors.joining(", ")); + .map(JsonElement::getAsJsonObject) + .map(jo -> jo.get("raw_author_name")) + .filter(Predicate.not(JsonElement::isJsonNull)) + .map(JsonElement::getAsString) + .collect(Collectors.joining(", ")); if (authors.isBlank()) { authors = null; } Integer publicationYear = Utils.integerOrNull(citationObject.get("publication_year")); - String crossrefType = Utils.stringOrNull(citationObject.get("type_crossref")); - MentionType mentionType = CrossrefMention.crossrefTypeMap.getOrDefault(crossrefType, MentionType.other); + String type = Utils.stringOrNull(citationObject.get("type")); + MentionType mentionType = openalexTypeMap.getOrDefault(type, MentionType.other); - String openalexIdString = citationObject - .getAsJsonObject("ids") - .getAsJsonPrimitive("openalex") - .getAsString(); + String openalexIdString = citationObject.getAsJsonObject("ids").getAsJsonPrimitive("openalex").getAsString(); OpenalexId openalexId = OpenalexId.fromString(openalexIdString); return new ExternalMentionRecord( - doi, - null, - openalexId, - url, - title, - authors, - null, - publicationYear, - null, - null, - mentionType, - "OpenAlex", - null + doi, + null, + openalexId, + url, + title, + authors, + null, + publicationYear, + null, + null, + mentionType, + "OpenAlex", + null ); } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenalexId.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenalexId.java index 03ac679..1645e87 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenalexId.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/OpenalexId.java @@ -5,16 +5,17 @@ package nl.esciencecenter.rsd.scraper.doi; -import nl.esciencecenter.rsd.scraper.Utils; - import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.Utils; // https://docs.openalex.org/how-to-use-the-api/get-single-entities#the-openalex-id public class OpenalexId { - private static final Pattern OPENALEX_PATTERN = Pattern.compile("^https://openalex\\.org/([WwAaSsIiCcPpFf]\\d{3,13})$"); + private static final Pattern OPENALEX_PATTERN = Pattern.compile( + "^https://openalex\\.org/([WwAaSsIiCcPpFf]\\d{3,13})$" + ); private static final String OPENALEX_ID_BASE = "https://openalex.org/"; private final String openalexKey; diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestCitationRepository.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestCitationRepository.java index cad4b1b..b10edac 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestCitationRepository.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestCitationRepository.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2023 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 @@ -9,14 +9,15 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.Utils; - import java.time.Instant; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Objects; import java.util.UUID; +import java.util.stream.Collectors; +import nl.esciencecenter.rsd.scraper.Utils; /** * This class provides access to the citation related tables via the Postgrest API. @@ -33,27 +34,50 @@ public PostgrestCitationRepository(String backendUrl) { /** * Retrieve the least recently scraped reference papers from the database. * - * @param limit the maximum number of references to return + * @param limit the maximum number of references to return, should be non-negative * @return A collection of citation data representing these reference papers. */ public Collection leastRecentlyScrapedCitations(int limit) { + if (limit < 0) { + throw new IllegalArgumentException("limit should be non-negative but was %d".formatted(limit)); + } + if (limit == 0) { + return Collections.emptyList(); + } + String oneHourAgoFilter = Utils.atLeastOneHourAgoFilter("citations_scraped_at"); - String uri = backendUrl + "/rpc/reference_papers_to_scrape?order=citations_scraped_at.asc.nullsfirst&limit=" + limit + "&" + oneHourAgoFilter; + String uri = + backendUrl + + "/rpc/reference_papers_to_scrape?order=citations_scraped_at.asc.nullsfirst&limit=" + + limit + + "&" + + oneHourAgoFilter; String data = Utils.getAsAdmin(uri); return parseJson(data); } - public void saveCitations(String backendUrl, UUID idCitedMention, Collection citingMentions, Instant scrapedAt) { + public void updateScrapedAtTime(Collection uuidsOfMentions, Instant scrapedAt) { + if (uuidsOfMentions.isEmpty()) { + return; + } + Objects.requireNonNull(scrapedAt); + + String orFilter = uuidsOfMentions + .stream() + .map("id.eq.%s"::formatted) + .collect(Collectors.joining(",", "or=(", ")")); + String jsonPatch = "{\"citations_scraped_at\": \"%s\"}".formatted(scrapedAt.toString()); - Utils.patchAsAdmin(backendUrl + "/mention?id=eq." + idCitedMention.toString(), jsonPatch); + Utils.patchAsAdmin(backendUrl + "/mention?" + orFilter, jsonPatch); + } + public void saveCitations(String backendUrl, UUID idCitedMention, Collection citingMentions) { JsonArray jsonArray = new JsonArray(); // We sometimes encounter duplicate citations which may lead to the operation to fail. HashSet seen = new HashSet<>(); for (UUID citingMention : citingMentions) { - if (citingMention != null) { String citationID = citingMention.toString(); @@ -73,7 +97,6 @@ public void saveCitations(String backendUrl, UUID idCitedMention, Collection parseJson(String data) { - JsonArray array = JsonParser.parseString(data).getAsJsonArray(); Collection result = new ArrayList<>(); diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestMentionRepository.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestMentionRepository.java index 37e62b6..05df578 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestMentionRepository.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestMentionRepository.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 @@ -14,8 +14,6 @@ import com.google.gson.JsonParser; import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializer; -import nl.esciencecenter.rsd.scraper.Utils; - import java.net.URI; import java.time.Instant; import java.time.ZonedDateTime; @@ -23,20 +21,29 @@ import java.util.Collection; import java.util.Objects; import java.util.UUID; +import nl.esciencecenter.rsd.scraper.Utils; public class PostgrestMentionRepository { private static final Gson GSON = new GsonBuilder() - .serializeNulls() - .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES) - .registerTypeAdapter(Instant.class, (JsonSerializer) (src, typeOfSrc, context) -> new JsonPrimitive(src.toString())) - .registerTypeAdapter(ZonedDateTime.class, (JsonSerializer) (src, typeOfSrc, context) -> new JsonPrimitive(src.toString())) - .registerTypeAdapter(Doi.class, (JsonSerializer) (src, typeOfSrc, context) -> new JsonPrimitive(src.toString())) - .create(); + .serializeNulls() + .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES) + .registerTypeAdapter( + Instant.class, + (JsonSerializer) (src, typeOfSrc, context) -> new JsonPrimitive(src.toString()) + ) + .registerTypeAdapter( + ZonedDateTime.class, + (JsonSerializer) (src, typeOfSrc, context) -> new JsonPrimitive(src.toString()) + ) + .registerTypeAdapter( + Doi.class, + (JsonSerializer) (src, typeOfSrc, context) -> new JsonPrimitive(src.toString()) + ) + .create(); private final String backendUrl; - public PostgrestMentionRepository(String backendUrl) { this.backendUrl = Objects.requireNonNull(backendUrl); } @@ -46,7 +53,6 @@ static Collection parseMultipleRsdIds(String json) { JsonArray rootArray = JsonParser.parseString(json).getAsJsonArray(); for (JsonElement jsonElement : rootArray) { - JsonObject arrayEntry = jsonElement.getAsJsonObject(); UUID id = UUID.fromString(arrayEntry.getAsJsonPrimitive("id").getAsString()); String doiString = Utils.stringOrNull(arrayEntry.get("doi")); @@ -77,27 +83,43 @@ static RsdMentionIds parseSingleRsdIds(String json) { } public Collection leastRecentlyScrapedMentions(int limit) { - String data = Utils.getAsAdmin(backendUrl + "/mention?or=(doi.not.is.null,openalex_id.not.is.null)&order=scraped_at.asc.nullsfirst&select=id,doi,openalex_id&limit=" + limit); + String data = Utils.getAsAdmin( + backendUrl + + "/mention?or=(doi.not.is.null,openalex_id.not.is.null)&order=scraped_at.asc.nullsfirst&select=id,doi,openalex_id&limit=" + + limit + ); return parseMultipleRsdIds(data); } public void saveScrapedAt(RsdMentionIds ids, Instant scrapedAt) { JsonObject root = new JsonObject(); root.addProperty("scraped_at", scrapedAt.toString()); - Utils.patchAsAdmin(backendUrl + "/mention?select=id,doi,openalex_id&id=eq." + ids.id(), root.toString(), "Prefer", "return=representation"); + Utils.patchAsAdmin( + backendUrl + "/mention?select=id,doi,openalex_id&id=eq." + ids.id(), + root.toString(), + "Prefer", + "return=representation" + ); } public RsdMentionIds updateMention(RsdMentionRecord mention, boolean updateOpenAlexId) { JsonObject root = createJsonFromMentionData(mention.content(), updateOpenAlexId); root.addProperty("scraped_at", mention.scrapedAt().toString()); - String response = Utils.patchAsAdmin(backendUrl + "/mention?select=id,doi,openalex_id&id=eq." + mention.id(), root.toString(), "Prefer", "return=representation"); + String response = Utils.patchAsAdmin( + backendUrl + "/mention?select=id,doi,openalex_id&id=eq." + mention.id(), + root.toString(), + "Prefer", + "return=representation" + ); return parseSingleRsdIds(response); } public RsdMentionIds createMentionIfNotExistsOnDoiAndGetIds(ExternalMentionRecord mention, Instant scrapedAt) { Doi doi = mention.doi(); Objects.requireNonNull(doi); - Collection mentionsWithDoi = parseMultipleRsdIds(Utils.getAsAdmin(backendUrl + "/mention?select=id,doi,openalex_id&doi=eq." + doi.toUrlEncodedString())); + Collection mentionsWithDoi = parseMultipleRsdIds( + Utils.getAsAdmin(backendUrl + "/mention?select=id,doi,openalex_id&doi=eq." + doi.toUrlEncodedString()) + ); if (mentionsWithDoi.size() == 1) { return mentionsWithDoi.iterator().next(); } @@ -111,7 +133,10 @@ public RsdMentionIds createOrUpdateMentionWithOpenalexId(ExternalMentionRecord m String query = "/mention?select=id,doi,openalex_id"; if (mention.doi() != null) { - query += "&or=(openalex_id.eq.%s,doi.eq.%s)".formatted(openalexId.toUrlEncodedString(), doi.toUrlEncodedString()); + query += "&or=(openalex_id.eq.%s,doi.eq.%s)".formatted( + openalexId.toUrlEncodedString(), + doi.toUrlEncodedString() + ); } else { query += "&openalex_id=eq.%s".formatted(openalexId.toUrlEncodedString()); } @@ -119,7 +144,12 @@ public RsdMentionIds createOrUpdateMentionWithOpenalexId(ExternalMentionRecord m Collection existingIds = parseMultipleRsdIds(existingMentionsResponse); if (existingIds.size() > 1) { - throw new RuntimeException("Multiple entries with DOI %s or OpenAlex id %s exist, they should be merged".formatted(openalexId, mention.doi())); + throw new RuntimeException( + "Multiple entries with DOI %s or OpenAlex id %s exist, they should be merged".formatted( + openalexId, + mention.doi() + ) + ); } if (existingIds.size() == 1) { UUID id = existingIds.iterator().next().id(); @@ -132,7 +162,12 @@ public RsdMentionIds createOrUpdateMentionWithOpenalexId(ExternalMentionRecord m private RsdMentionIds createNewMention(ExternalMentionRecord mention, Instant scrapedAt, boolean setOpenAlexId) { JsonObject root = createJsonFromMentionData(mention, setOpenAlexId); root.addProperty("scraped_at", scrapedAt.toString()); - String response = Utils.postAsAdmin(backendUrl + "/mention?select=id,doi,openalex_id", root.toString(), "Prefer", "return=representation"); + String response = Utils.postAsAdmin( + backendUrl + "/mention?select=id,doi,openalex_id", + root.toString(), + "Prefer", + "return=representation" + ); return parseSingleRsdIds(response); } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestReleaseRepository.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestReleaseRepository.java index 51e13c3..fa86bc3 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestReleaseRepository.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/PostgrestReleaseRepository.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2023 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 @@ -12,9 +12,6 @@ import com.google.gson.JsonDeserializer; import com.google.gson.JsonObject; import com.google.gson.reflect.TypeToken; -import nl.esciencecenter.rsd.scraper.Config; -import nl.esciencecenter.rsd.scraper.Utils; - import java.time.Instant; import java.util.ArrayList; import java.util.Collection; @@ -22,6 +19,8 @@ import java.util.Map; import java.util.Objects; import java.util.UUID; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.Utils; public class PostgrestReleaseRepository { @@ -32,11 +31,19 @@ public PostgrestReleaseRepository(String backendUrl) { } public Collection leastRecentlyScrapedReleases(int limit) { - String data = Utils.getAsAdmin(backendUrl + "/rpc/software_join_release?concept_doi=not.is.null&order=releases_scraped_at.asc.nullsfirst&limit=" + limit); + String data = Utils.getAsAdmin( + backendUrl + + "/rpc/software_join_release?concept_doi=not.is.null&order=releases_scraped_at.asc.nullsfirst&limit=" + + limit + ); return parseJson(data); } - public void saveReleaseContent(Collection releaseData, Map> conceptDoiToDois, Map versionDoiToMentionId) { + public void saveReleaseContent( + Collection releaseData, + Map> conceptDoiToDois, + Map versionDoiToMentionId + ) { // First update the releases_scraped_at column. JsonArray releasesBody = new JsonArray(); Instant now = Instant.now(); @@ -46,14 +53,20 @@ public void saveReleaseContent(Collection releaseData, Map> conceptDoiToSoftwareIds = new HashMap<>(); for (ReleaseData release : releaseData) { - Collection softwareIds = conceptDoiToSoftwareIds.computeIfAbsent(release.conceptDoi, k -> new ArrayList<>()); + Collection softwareIds = conceptDoiToSoftwareIds.computeIfAbsent(release.conceptDoi, k -> + new ArrayList<>() + ); softwareIds.add(release.softwareId); } @@ -83,17 +96,22 @@ public void saveReleaseContent(Collection releaseData, Map parseJson(String data) { - JsonDeserializer doiDeserializer = (json, type, context) -> Doi.fromString(json.getAsJsonPrimitive().getAsString()); + JsonDeserializer doiDeserializer = (json, type, context) -> + Doi.fromString(json.getAsJsonPrimitive().getAsString()); Gson gson = new GsonBuilder() - .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES) - .registerTypeAdapter(Doi.class, doiDeserializer) - .create(); - TypeToken> typeToken = new TypeToken<>() { - }; + .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES) + .registerTypeAdapter(Doi.class, doiDeserializer) + .create(); + TypeToken> typeToken = new TypeToken<>() {}; return gson.fromJson(data, typeToken.getType()); } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ReleaseData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ReleaseData.java index 55807be..746877b 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ReleaseData.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/ReleaseData.java @@ -17,11 +17,19 @@ public class ReleaseData { @Override public String toString() { - return "ReleaseData{" + - "softwareId=" + softwareId + - ", slug='" + slug + '\'' + - ", conceptDoi='" + conceptDoi + '\'' + - ", versionedDois=" + versionedDois + - '}'; + return ( + "ReleaseData{" + + "softwareId=" + + softwareId + + ", slug='" + + slug + + '\'' + + ", conceptDoi='" + + conceptDoi + + '\'' + + ", versionedDois=" + + versionedDois + + '}' + ); } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionIds.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionIds.java index 81d529e..36976ce 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionIds.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionIds.java @@ -7,9 +7,4 @@ import java.util.UUID; -public record RsdMentionIds( - UUID id, - Doi doi, - OpenalexId openalexId -) { -} +public record RsdMentionIds(UUID id, Doi doi, OpenalexId openalexId) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionRecord.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionRecord.java index 237e961..4df9178 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionRecord.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/RsdMentionRecord.java @@ -8,9 +8,4 @@ import java.time.Instant; import java.util.UUID; -public record RsdMentionRecord( - UUID id, - ExternalMentionRecord content, - Instant scrapedAt -) { -} +public record RsdMentionRecord(UUID id, ExternalMentionRecord content, Instant scrapedAt) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/EscapeOssrSoftwareEntry.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/EscapeOssrSoftwareEntry.java new file mode 100644 index 0000000..8398f10 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/EscapeOssrSoftwareEntry.java @@ -0,0 +1,385 @@ +// SPDX-FileCopyrightText: 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2026 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.escape_ossr; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; +import io.xberg.htmltomarkdown.ConversionOptions; +import io.xberg.htmltomarkdown.HtmlToMarkdown; +import io.xberg.htmltomarkdown.OutputFormat; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.RsdContributor; +import nl.esciencecenter.rsd.scraper.RsdParsingException; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; +import nl.esciencecenter.rsd.scraper.doi.Doi; +import nl.esciencecenter.rsd.scraper.git.CodePlatformProvider; +import nl.esciencecenter.rsd.scraper.git.GithubScraper; +import nl.esciencecenter.rsd.scraper.git.GitlabScraper; +import nl.esciencecenter.rsd.scraper.license.SpdxLicense; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EscapeOssrSoftwareEntry { + + private static final Logger LOGGER = LoggerFactory.getLogger(EscapeOssrSoftwareEntry.class); + + private static final Pattern ORCID_PATTERN = Pattern.compile("\\d{4}-\\d{4}-\\d{4}-\\d{3}[0-9X]"); + + final String name; + final Doi conceptDoi; + final String shortDescription; + final String longDescription; + Optional descriptionUrl = Optional.empty(); + final Optional getStartedUrl; + final String repoUrl; + final CodePlatformProvider codePlatform; + final List keywords; + final List contributors; + final Optional license; + final List applicationCategories; + final List applicationSubCategories; + + private EscapeOssrSoftwareEntry( + String name, + Doi conceptDoi, + String shortDescription, + String longDescription, + Optional getStartedUrl, + String repoUrl, + CodePlatformProvider codePlatform, + List keywords, + List contributors, + Optional license, + List applicationCategories, + List applicationSubCategories + ) { + this.name = name; + this.conceptDoi = conceptDoi; + this.shortDescription = shortDescription; + this.longDescription = longDescription; + this.getStartedUrl = getStartedUrl; + this.repoUrl = repoUrl; + this.codePlatform = codePlatform; + this.keywords = keywords; + this.contributors = contributors; + this.license = license; + this.applicationCategories = applicationCategories; + this.applicationSubCategories = applicationSubCategories; + } + + public static EscapeOssrSoftwareEntry fromCodemeta(String codemeta, Doi conceptDoi) throws RsdParsingException { + try { + JsonObject root = JsonParser.parseString(codemeta).getAsJsonObject(); + + String name = root.getAsJsonPrimitive("name").getAsString(); + + String description = root.getAsJsonPrimitive("description").getAsString(); + String shortDescription = description; + try { + shortDescription = HtmlToMarkdown.convert( + description, + ConversionOptions.builder().withOutputFormat(OutputFormat.Plain).withSkipImages(true).build() + ).content(); + } catch (Exception e) { + LOGGER.warn("Could not convert description as HTML to plaintext", e); + } + String longDescription = description; + try { + longDescription = HtmlToMarkdown.convert( + description, + ConversionOptions.builder().withOutputFormat(OutputFormat.Markdown).build() + ).content(); + } catch (Exception e) { + LOGGER.warn("Could not convert description as HTML to Markdown", e); + } + + Optional getStartedUrl = Optional.empty(); + JsonElement readmeJson = root.get("readme"); + if (readmeJson != null && readmeJson.isJsonPrimitive()) { + String readmeUrlToCheck = readmeJson.getAsJsonPrimitive().getAsString(); + if (Utils.isUrl(readmeUrlToCheck)) { + getStartedUrl = Optional.of(readmeUrlToCheck); + } + } + + String repoUrl = root.getAsJsonPrimitive("codeRepository").getAsString(); + if (repoUrl.startsWith("git+")) { + repoUrl = repoUrl.substring(4); + } + if (repoUrl.endsWith(".git")) { + repoUrl = repoUrl.substring(0, repoUrl.length() - 4); + } + CodePlatformProvider codePlatform = CodePlatformProvider.OTHER; + if (repoUrl.startsWith("https://github.com")) { + codePlatform = CodePlatformProvider.GITHUB; + } else if (repoUrl.contains("gitlab")) { + codePlatform = CodePlatformProvider.GITLAB; + } else if ( + repoUrl.startsWith("https://git.ligo.org") || + repoUrl.startsWith("https://git.astron.nl") || + repoUrl.startsWith("https://baltig.infn.it") + ) { + codePlatform = CodePlatformProvider.GITLAB; + } + + JsonElement keywordsJson = root.get("keywords"); + List keywords = new ArrayList<>(); + if (keywordsJson != null && keywordsJson.isJsonArray()) { + for (JsonElement keywordJson : keywordsJson.getAsJsonArray()) { + keywords.add(keywordJson.getAsString()); + } + } + + List contributors = new ArrayList<>(); + contributors.addAll(parseContributorArray(root.get("author"), "author", 1)); + contributors.addAll(parseContributorArray(root.get("contributor"), "contributor", contributors.size() + 1)); + contributors.addAll(parseContributorArray(root.get("maintainer"), "maintainer", contributors.size() + 1)); + + Optional license = Optional.empty(); + JsonElement licenseJson = root.get("license"); + if (licenseJson != null && licenseJson.isJsonPrimitive()) { + license = Optional.of(licenseJson.getAsString()); + } + + List applicationCategories = new ArrayList<>(); + JsonElement applicationCategoriesJson = root.get("applicationCategory"); + if (applicationCategoriesJson != null && applicationCategoriesJson.isJsonPrimitive()) { + applicationCategories.addAll(Arrays.asList(applicationCategoriesJson.getAsString().split(", "))); + } + + List applicationSubCategories = new ArrayList<>(); + JsonElement applicationSubCategoriesJson = root.get("applicationSubCategory"); + if (applicationSubCategoriesJson != null && applicationSubCategoriesJson.isJsonPrimitive()) { + applicationCategories.addAll(Arrays.asList(applicationSubCategoriesJson.getAsString().split(", "))); + } + + return new EscapeOssrSoftwareEntry( + name, + conceptDoi, + shortDescription, + longDescription, + getStartedUrl, + repoUrl, + codePlatform, + keywords, + contributors, + license, + applicationCategories, + applicationSubCategories + ); + } catch (RuntimeException e) { + throw new RsdParsingException("Could no parse CodeMeta", e); + } + } + + /** + * + * Retrieve the raw README URL of the repository and store it in this object, overwriting the previous result (if it existed). + * + * @throws RsdResponseException when the HTTP request to the API of the repo gives an unexpected response + * @throws IOException when the HTTP request to the API throws this + * @throws InterruptedException when the HTTP request to the API is interrupted + */ + public void harvestReadmeUrl() throws RsdResponseException, IOException, InterruptedException { + Optional rawReadmeUrl = switch (codePlatform) { + case GITHUB -> { + Optional githubScraper = GithubScraper.create(repoUrl); + if (githubScraper.isEmpty()) { + yield Optional.empty(); + } + + yield githubScraper.get().rawReadmeUrl(); + } + case GITLAB -> { + URI uri = URI.create(repoUrl); + String path = uri.getRawPath(); + if (path.startsWith("/")) { + path = path.substring(1); + } + if (path.endsWith("/")) { + path = path.substring(0, path.length() - 1); + } + + GitlabScraper gitScraper = new GitlabScraper("https://" + uri.getHost() + "/api", path); + yield gitScraper.rawReadmeUrl(); + } + case null, default -> Optional.empty(); + }; + + this.descriptionUrl = rawReadmeUrl.map(URI::toASCIIString).filter(url -> url.endsWith(".md")); + } + + private static List parseContributorArray(JsonElement element, String role, int startPosition) { + if (element == null || !element.isJsonArray()) { + return Collections.emptyList(); + } + + JsonArray array = element.getAsJsonArray(); + List result = new ArrayList<>(array.size()); + + for (JsonElement jsonElement : array) { + JsonObject authorObject = jsonElement.getAsJsonObject(); + if (!authorObject.getAsJsonPrimitive("@type").getAsString().equals("Person")) { + continue; + } + + String givenNames = authorObject.getAsJsonPrimitive("givenName").getAsString(); + String familyNames = authorObject.getAsJsonPrimitive("familyName").getAsString(); + + Optional affiliation = Optional.empty(); + if (authorObject.get("affiliation") instanceof JsonPrimitive affiliationJson) { + affiliation = Optional.of(affiliationJson.getAsString()); + } else if (authorObject.get("affiliation") instanceof JsonObject affiliationJson) { + affiliation = Optional.of(affiliationJson.getAsJsonPrimitive("name").getAsString()); + } + + Optional orcid = parseOrcid(authorObject.get("@id")); + if (orcid.isEmpty()) { + orcid = parseOrcid(authorObject.get("identifier")); + } + + RsdContributor contributor = new RsdContributor( + givenNames, + familyNames, + affiliation, + Optional.of(role), + orcid, + startPosition + ); + ++startPosition; + result.add(contributor); + } + + return result; + } + + private static Optional parseOrcid(JsonElement element) { + if (element == null || !element.isJsonPrimitive()) { + return Optional.empty(); + } + + Matcher orcidMatcher = ORCID_PATTERN.matcher(element.getAsString()); + if (orcidMatcher.find()) { + return Optional.of(orcidMatcher.group(0)); + } + + return Optional.empty(); + } + + public String toRsdJson(Map licenseMap) { + JsonObject root = new JsonObject(); + root.addProperty("community_slug", "escape-ossr"); + root.addProperty("slug", "escape-ossr-" + Utils.slugify(name)); + root.addProperty("brand_name", name); + root.addProperty("concept_doi", conceptDoi.toString()); + root.addProperty( + "short_statement", + shortDescription.length() > 300 ? shortDescription.substring(0, 297) + "..." : shortDescription + ); + root.addProperty("description", longDescription); + root.addProperty("description_url", descriptionUrl.orElse(null)); + root.addProperty("get_started_url", getStartedUrl.orElse(null)); + + root.addProperty("repository_url", repoUrl); + root.addProperty("code_platform", codePlatform.toDatabaseString()); + root.add("scraping_disabled_reason", JsonNull.INSTANCE); + + JsonArray keywordsArray = new JsonArray(keywords.size()); + for (String keyword : keywords) { + keywordsArray.add(keyword); + } + root.add("keywords_array", keywordsArray); + + JsonArray familyNamesArray = new JsonArray(contributors.size()); + JsonArray givenNamesArray = new JsonArray(contributors.size()); + JsonArray roleArray = new JsonArray(contributors.size()); + JsonArray affiliationArray = new JsonArray(contributors.size()); + JsonArray orcidArray = new JsonArray(contributors.size()); + JsonArray positionArray = new JsonArray(contributors.size()); + + for (RsdContributor contributor : contributors) { + familyNamesArray.add(contributor.familyNames()); + givenNamesArray.add(contributor.givenNames()); + affiliationArray.add(contributor.affiliation().map(Utils::sanitiseWhitespace).orElse(null)); + roleArray.add(contributor.role().orElseThrow()); + orcidArray.add(contributor.orcid().orElse(null)); + positionArray.add(contributor.position()); + } + + if (license.isPresent()) { + String rawLicense = license.get(); + Optional optionalSpdxLicense = extractSpdxLicense(rawLicense, licenseMap); + if (optionalSpdxLicense.isPresent()) { + SpdxLicense spdxLicense = optionalSpdxLicense.get(); + + root.addProperty("license_value", spdxLicense.licenseId()); + root.addProperty("license_name", spdxLicense.name()); + root.addProperty("license_url", spdxLicense.reference()); + root.addProperty("license_open_source", true); + } else { + root.add("license_value", JsonNull.INSTANCE); + root.add("license_name", JsonNull.INSTANCE); + root.add("license_url", JsonNull.INSTANCE); + root.add("license_open_source", JsonNull.INSTANCE); + } + } + + root.add("family_names_array", familyNamesArray); + root.add("given_names_array", givenNamesArray); + root.add("affiliation_array", affiliationArray); + root.add("role_array", roleArray); + root.add("orcid_array", orcidArray); + root.add("position_array", positionArray); + + JsonObject categoryObject = new JsonObject(); + root.add("categories", categoryObject); + + JsonArray applicationCategoryValues = new JsonArray(); + applicationCategories.forEach(applicationCategoryValues::add); + categoryObject.add("Application category", applicationCategoryValues); + + JsonArray applicationSubCategoryValues = new JsonArray(); + applicationSubCategories.forEach(applicationSubCategoryValues::add); + categoryObject.add("Application sub category", applicationSubCategoryValues); + + root.add("related_modules", new JsonArray()); + root.add("regular_mentions", new JsonArray()); + + return root.toString(); + } + + static Optional extractSpdxLicense(String rawLicense, Map licenseMap) { + if (rawLicense == null) { + return Optional.empty(); + } + + if (licenseMap.containsKey(rawLicense)) { + return Optional.of(licenseMap.get(rawLicense)); + } + + String urlPrefix = "https://spdx.org/licenses/"; + if (!rawLicense.startsWith(urlPrefix)) { + return Optional.empty(); + } + + String spdxId = rawLicense.substring(urlPrefix.length()); + return Optional.ofNullable(licenseMap.get(spdxId)); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/MainEscapeOssr.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/MainEscapeOssr.java new file mode 100644 index 0000000..bfab25b --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/MainEscapeOssr.java @@ -0,0 +1,148 @@ +// SPDX-FileCopyrightText: 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2026 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.escape_ossr; + +import java.io.IOException; +import java.net.URI; +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; +import nl.esciencecenter.rsd.scraper.license.GitHubSpdxLicenseRepository; +import nl.esciencecenter.rsd.scraper.license.SpdxLicense; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MainEscapeOssr { + + private static final Logger LOGGER = LoggerFactory.getLogger(MainEscapeOssr.class); + private static final String SERVICE_NAME = "ESCAPE_OSSR harvester"; + + public static void main() { + if (!Config.isEscapeOssrScraperEnabled()) { + LOGGER.info("The ESCAPE OSSR harvester is disabled"); + return; + } + + LOGGER.info("Start harvesting ESCAPE OSSR software"); + + final Map licenseMap; + final Collection escapeOssrRecords; + try { + LOGGER.info("Downloading SPDX licenses..."); + licenseMap = GitHubSpdxLicenseRepository.getLicensesByIdMap(); + LOGGER.info("Downloaded SPDX licenses successfully"); + + LOGGER.info("Downloading metadata of ESCAPE OSSR records..."); + escapeOssrRecords = ZenodoConnector.getEscapeOssrRecords(); + LOGGER.info("Found {} ESCAPE OSSR records", escapeOssrRecords.size()); + } catch (RsdResponseException e) { + LOGGER.error("Error when making request to {}, got status {} and body {}", e.uri, e.statusCode, e.body, e); + Utils.saveExceptionInDatabase(SERVICE_NAME, null, null, e); + return; + } catch (Exception e) { + LOGGER.error("Error", e); + Utils.saveExceptionInDatabase(SERVICE_NAME, null, null, e); + + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + + return; + } + + try (ExecutorService threadpool = Executors.newCachedThreadPool()) { + for (ZenodoEntry escapeOssrRecord : escapeOssrRecords) { + Runnable task = () -> { + try { + LOGGER.info("Trying to find CodeMeta file URL for entry {}", escapeOssrRecord.id()); + Optional codemetaUrl = ZenodoConnector.getCodemetaUrl(escapeOssrRecord); + if (codemetaUrl.isEmpty()) { + LOGGER.warn( + "No CodeMeta file found for entry {}, the files link was {}", + escapeOssrRecord.id(), + escapeOssrRecord.filesLink() + ); + return; + } + + LOGGER.info( + "Found CodeMeta file URL for entry {}, starting download...", + escapeOssrRecord.id() + ); + + String codemetaContent = ZenodoConnector.downloadCodemeta(codemetaUrl.get()); + LOGGER.info( + "Successfully downloaded CodeMeta file for {}, converting to RSD entry", + escapeOssrRecord.id() + ); + + EscapeOssrSoftwareEntry softwareEntry = EscapeOssrSoftwareEntry.fromCodemeta( + codemetaContent, + escapeOssrRecord.conceptDoi() + ); + try { + softwareEntry.harvestReadmeUrl(); + } catch (RsdResponseException e) { + LOGGER.warn( + "Exception when getting README URL of {}, using description from CodeMeta instead, made request to {}, got status {} and body {}", + softwareEntry.repoUrl, + e.uri, + e.statusCode, + e.body, + e + ); + } catch (IOException e) { + LOGGER.warn( + "Exception when getting README URL of {}, using description from CodeMeta instead", + softwareEntry.repoUrl, + e + ); + } + + LOGGER.info("Saving RSD entry for {} in the database", escapeOssrRecord.id()); + Utils.postAsAdmin( + Config.backendBaseUrl() + "/rpc/software_import", + softwareEntry.toRsdJson(licenseMap) + ); + LOGGER.info("Saved {} as entry in the database", escapeOssrRecord.id()); + } catch (RsdResponseException e) { + LOGGER.error( + "Error for entry {}, files URL {}, when making request to {}, got status {} and body {}", + escapeOssrRecord.id(), + escapeOssrRecord.filesLink(), + e.uri, + e.statusCode, + e.body, + e + ); + Utils.saveExceptionInDatabase(SERVICE_NAME, null, null, e); + } catch (Exception e) { + LOGGER.error( + "Error for entry {}, files URL {}", + escapeOssrRecord.id(), + escapeOssrRecord.filesLink(), + e + ); + Utils.saveExceptionInDatabase(SERVICE_NAME, null, null, e); + + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + } + }; + + threadpool.submit(task); + } + } + + LOGGER.info("Done harvesting ESCAPE OSSR software"); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/ZenodoConnector.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/ZenodoConnector.java new file mode 100644 index 0000000..d5b0509 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/ZenodoConnector.java @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2026 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.escape_ossr; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpResponse; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import nl.esciencecenter.rsd.scraper.RsdParsingException; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; + +public class ZenodoConnector { + + private static final String ESCAPE_OSSR_RECORDS_URL = + "https://zenodo.org/api/records?communities=escape2020&type=software"; + + private ZenodoConnector() {} + + public static Collection getEscapeOssrRecords() + throws IOException, InterruptedException, RsdParsingException, RsdResponseException { + int page = 1; + int pageSize = 25; + AtomicBoolean isLastPage = new AtomicBoolean(false); + Collection result = new ArrayList<>(); + + while (!isLastPage.getPlain()) { + HttpResponse response = Utils.getAsHttpResponse( + ESCAPE_OSSR_RECORDS_URL + "&page=%d&size=%d".formatted(page, pageSize) + ); + + if (response.statusCode() != 200) { + throw new RsdResponseException( + response.statusCode(), + response.uri(), + response.body(), + "Could not query Escape-OSSR records" + ); + } + + result.addAll(ZenodoParser.parseZenodoRecords(response.body(), isLastPage, pageSize)); + ++page; + } + + // this is a wrong, duplicate entry of "Gammapy: Python toolbox for gamma-ray astronomy", + // the correct entry has ID 20432079 + result.removeIf(entry -> entry.id().equals("14867036")); + + return result; + } + + public static Optional getCodemetaUrl(ZenodoEntry zenodoEntry) + throws IOException, InterruptedException, RsdParsingException, RsdResponseException { + HttpResponse filesResponse = Utils.getAsHttpResponse(zenodoEntry.filesLink()); + + if (filesResponse.statusCode() != 200) { + throw new RsdResponseException( + filesResponse.statusCode(), + filesResponse.uri(), + filesResponse.body(), + "Could not query Zenodo files" + ); + } + + Collection files = ZenodoParser.parseZenodoFileMetadataEntries(filesResponse.body()); + + Optional directCodemetaUrl = files + .stream() + .filter(entry -> "application/json".equals(entry.mimetype())) + .filter(entry -> "codemeta.json".equals(entry.key())) + .findAny() + .map(ZenodoFileMetadata::contentUrl); + if (directCodemetaUrl.isPresent()) { + return directCodemetaUrl; + } + + Collection zipFiles = files + .stream() + .filter(entry -> "application/zip".equals(entry.mimetype())) + .filter(entry -> entry.containerUrl().isPresent()) + .toList(); + + for (ZenodoFileMetadata zipFileMetadata : zipFiles) { + HttpResponse zipContainerResponse = Utils.getAsHttpResponse( + zipFileMetadata.containerUrl().orElseThrow() + ); + + if (zipContainerResponse.statusCode() != 200) { + throw new RsdResponseException( + zipContainerResponse.statusCode(), + zipContainerResponse.uri(), + zipContainerResponse.body(), + "Could not query Zenodo container link" + ); + } + + Collection zipEntries = ZenodoParser.parseZenodoZipFileMetadataEntries( + zipContainerResponse.body() + ); + + Optional zippedCodemetaUrl = zipEntries + .stream() + .filter(entry -> "application/json".equals(entry.mimetype())) + .filter(entry -> entry.key() != null && entry.key().endsWith("codemeta.json")) + .findAny() + .map(ZenodoZipEntryMetadata::contentUrl); + if (zippedCodemetaUrl.isPresent()) { + return zippedCodemetaUrl; + } + } + + return Optional.empty(); + } + + public static String downloadCodemeta(URI codemetaUrl) + throws IOException, InterruptedException, RsdResponseException { + HttpResponse filesResponse = Utils.getAsHttpResponse(codemetaUrl); + + if (filesResponse.statusCode() != 200) { + throw new RsdResponseException( + filesResponse.statusCode(), + filesResponse.uri(), + filesResponse.body(), + "Could not download CodeMeta" + ); + } + + return filesResponse.body(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/ZenodoEntry.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/ZenodoEntry.java new file mode 100644 index 0000000..5cfc22d --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/ZenodoEntry.java @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2026 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.escape_ossr; + +import java.net.URI; +import nl.esciencecenter.rsd.scraper.doi.Doi; + +public record ZenodoEntry(String id, Doi conceptDoi, URI filesLink) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/ZenodoFileMetadata.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/ZenodoFileMetadata.java new file mode 100644 index 0000000..8d19af6 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/ZenodoFileMetadata.java @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2026 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.escape_ossr; + +import java.net.URI; +import java.util.Optional; + +public record ZenodoFileMetadata(String key, String mimetype, URI contentUrl, Optional containerUrl) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/ZenodoParser.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/ZenodoParser.java new file mode 100644 index 0000000..c559ac6 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/ZenodoParser.java @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2026 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.escape_ossr; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import nl.esciencecenter.rsd.scraper.RsdParsingException; +import nl.esciencecenter.rsd.scraper.doi.Doi; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ZenodoParser { + + private static final Logger LOGGER = LoggerFactory.getLogger(ZenodoParser.class); + + private ZenodoParser() {} + + static Collection parseZenodoRecords(String json, AtomicBoolean isLastPage, int pageSize) + throws RsdParsingException { + try { + JsonObject responseObject = JsonParser.parseString(json).getAsJsonObject(); + JsonObject hitsObject = responseObject.getAsJsonObject("hits"); + JsonArray hitsArray = hitsObject.getAsJsonArray("hits"); + Collection result = new ArrayList<>(hitsArray.size()); + + if (hitsArray.size() < pageSize) { + isLastPage.setPlain(true); + } + + for (JsonElement jsonElement : hitsArray) { + try { + ZenodoEntry parsedRecord = parseZenodoRecord(jsonElement); + result.add(parsedRecord); + } catch (RsdParsingException e) { + LOGGER.warn("Skipping Zenodo entry with parsing errors", e); + } + } + + return result; + } catch (RuntimeException e) { + throw new RsdParsingException("Could not parse Zenodo records", e); + } + } + + static ZenodoEntry parseZenodoRecord(JsonElement zenodoElement) throws RsdParsingException { + try { + JsonObject zenodoObject = zenodoElement.getAsJsonObject(); + + String id = zenodoObject.getAsJsonPrimitive("id").getAsString(); + String rawConceptDoi = zenodoObject.getAsJsonPrimitive("conceptdoi").getAsString(); + Doi conceptDoi = Doi.fromString(rawConceptDoi); + + JsonObject linksArray = zenodoObject.getAsJsonObject("links"); + String filesLink = linksArray.getAsJsonPrimitive("files").getAsString(); + URI filesUrl = URI.create(filesLink); + + return new ZenodoEntry(id, conceptDoi, filesUrl); + } catch (RuntimeException e) { + throw new RsdParsingException("Error parsing Zenodo entry", e); + } + } + + static Collection parseZenodoFileMetadataEntries(String json) throws RsdParsingException { + try { + JsonObject responseObject = JsonParser.parseString(json).getAsJsonObject(); + JsonArray entriesArray = responseObject.getAsJsonArray("entries"); + Collection result = new ArrayList<>(entriesArray.size()); + + for (JsonElement jsonElement : entriesArray) { + try { + result.add(parseZenodoFileMetadataEntry(jsonElement)); + } catch (RsdParsingException e) { + LOGGER.warn("Skipping Zenodo file entry with parsing errors", e); + } + } + + return result; + } catch (RuntimeException e) { + throw new RsdParsingException("Could not parse Zenodo files", e); + } + } + + static ZenodoFileMetadata parseZenodoFileMetadataEntry(JsonElement fileElement) throws RsdParsingException { + try { + JsonObject fileObject = fileElement.getAsJsonObject(); + String key = fileObject.getAsJsonPrimitive("key").getAsString(); + String mimeType = fileObject.getAsJsonPrimitive("mimetype").getAsString(); + URI contentUrl = URI.create( + fileObject.getAsJsonObject("links").getAsJsonPrimitive("content").getAsString() + ); + + JsonElement containerElement = fileObject.getAsJsonObject("links").getAsJsonPrimitive("container"); + Optional containerUrl = + containerElement != null && containerElement.isJsonPrimitive() + ? Optional.of(URI.create(containerElement.getAsString())) + : Optional.empty(); + + return new ZenodoFileMetadata(key, mimeType, contentUrl, containerUrl); + } catch (RuntimeException e) { + throw new RsdParsingException("Error parsing Zenodo file entry", e); + } + } + + static Collection parseZenodoZipFileMetadataEntries(String json) + throws RsdParsingException { + try { + JsonObject responseObject = JsonParser.parseString(json).getAsJsonObject(); + JsonArray entriesArray = responseObject.getAsJsonArray("entries"); + Collection result = new ArrayList<>(entriesArray.size()); + + for (JsonElement jsonElement : entriesArray) { + try { + result.add(parseZenodoZipFileMetadataEntry(jsonElement)); + } catch (RsdParsingException e) { + LOGGER.warn("Skipping Zenodo zip file entry with parsing errors", e); + } + } + + return result; + } catch (RuntimeException e) { + throw new RsdParsingException("Could not parse Zenodo zip file entries", e); + } + } + + static ZenodoZipEntryMetadata parseZenodoZipFileMetadataEntry(JsonElement fileElement) throws RsdParsingException { + try { + JsonObject fileObject = fileElement.getAsJsonObject(); + String key = fileObject.getAsJsonPrimitive("key").getAsString(); + String mimeType = fileObject.getAsJsonPrimitive("mimetype").getAsString(); + URI contentUrl = URI.create( + fileObject.getAsJsonObject("links").getAsJsonPrimitive("content").getAsString() + ); + + return new ZenodoZipEntryMetadata(key, mimeType, contentUrl); + } catch (RuntimeException e) { + throw new RsdParsingException("Error parsing Zenodo zip file entry", e); + } + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/ZenodoZipEntryMetadata.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/ZenodoZipEntryMetadata.java new file mode 100644 index 0000000..0b6d173 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/escape_ossr/ZenodoZipEntryMetadata.java @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2026 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.escape_ossr; + +import java.net.URI; + +public record ZenodoZipEntryMetadata(String key, String mimetype, URI contentUrl) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitData.java index 2c93ed9..8dfa7c1 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitData.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitData.java @@ -11,5 +11,4 @@ public record BasicGitData( Long starCount, Integer forkCount, Integer openIssueCount -) { -} +) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitDatabaseData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitDatabaseData.java index 3374cc1..c163f49 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitDatabaseData.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicGitDatabaseData.java @@ -8,8 +8,7 @@ import java.time.ZonedDateTime; public record BasicGitDatabaseData( - BasicRepositoryData basicData, - BasicGitData statsData, - ZonedDateTime dataScrapedAt -) { -} + BasicRepositoryData basicData, + BasicGitData statsData, + ZonedDateTime dataScrapedAt +) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryData.java index 161fe4f..d0e66c7 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryData.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryData.java @@ -7,5 +7,4 @@ import java.util.UUID; -public record BasicRepositoryData(UUID software, String url) { -} +public record BasicRepositoryData(UUID software, String url) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryDataWithHistory.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryDataWithHistory.java index 39c4863..0296d44 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryDataWithHistory.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/BasicRepositoryDataWithHistory.java @@ -8,5 +8,9 @@ import java.time.ZonedDateTime; import java.util.UUID; -public record BasicRepositoryDataWithHistory(UUID software, String url, ZonedDateTime commitHistoryScrapedAt, CommitsPerWeek commitsPerWeek) { -} +public record BasicRepositoryDataWithHistory( + UUID software, + String url, + ZonedDateTime commitHistoryScrapedAt, + CommitsPerWeek commitsPerWeek +) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CodePlatformProvider.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CodePlatformProvider.java index 6ee6449..365d584 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CodePlatformProvider.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CodePlatformProvider.java @@ -1,10 +1,29 @@ -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.git; +import java.util.Locale; + public enum CodePlatformProvider { - GITHUB, GITLAB, BITBUCKET, FOURTU, OTHER + GITHUB, + GITLAB, + BITBUCKET, + FOURTU, + CODEBERG, + OTHER; + + /** + * + * @return a string matching the enum "platform_type" in the database + */ + public String toDatabaseString() { + if (this == FOURTU) { + return "4tu"; + } + + return this.name().toLowerCase(Locale.ENGLISH); + } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CodebergScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CodebergScraper.java new file mode 100644 index 0000000..79e3529 --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CodebergScraper.java @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.io.IOException; +import java.net.http.HttpResponse; +import java.time.ZonedDateTime; +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; + +public class CodebergScraper implements GitScraper { + + private static final Pattern codebergUrlPattern = Pattern.compile( + "^https://([^\\s/]+)/([^\\s/]+)/([^\\s/]+)(\\.git)?$" + ); + + private final String baseApiUrl; + private final CommitsPerWeek commitsPerWeek; + + /** + * This should only be called when not calling {@link #contributions()} later. + * + * @param url the Codeberg URL from which this scraper should be created, should have the form {@code https:////} + * @return an instance of {@link CodebergScraper} if the url is valid + */ + public static Optional fromUrl(String url) { + return fromUrl(url, new CommitsPerWeek()); + } + + /** + * This must be called when calling {@link #contributions()} later. + * + * @param url the Codeberg URL from which this scraper should be created, should have the form {@code https:////} + * @param commitsPerWeek the {@link CommitsPerWeek} to which commits should be appended to when calling {@link #contributions()} + * @return an instance of {@link CodebergScraper} if the url is valid and commitsPerWeek is not {@code null} + */ + public static Optional fromUrl(String url, CommitsPerWeek commitsPerWeek) { + if (url == null || commitsPerWeek == null) { + return Optional.empty(); + } + + Matcher urlMatcher = codebergUrlPattern.matcher(url); + if (!urlMatcher.find()) { + return Optional.empty(); + } + + String baseUrl = "https://" + urlMatcher.group(1); + String owner = urlMatcher.group(2); + String repo = urlMatcher.group(3); + + return Optional.of(new CodebergScraper(baseUrl, owner, repo, commitsPerWeek)); + } + + private CodebergScraper(String baseUrl, String owner, String repo, CommitsPerWeek commitsPerWeek) { + baseApiUrl = baseUrl + "/api/v1/repos/" + Utils.urlEncode(owner) + "/" + Utils.urlEncode(repo); + this.commitsPerWeek = Objects.requireNonNull(commitsPerWeek); + } + + // e.g. https://codeberg.org/api/v1/repos/privacyguides/privacyguides.org + @Override + public BasicGitData basicData() throws IOException, InterruptedException, RsdResponseException { + String response = Utils.get(baseApiUrl); + + return parseBasicData(response); + } + + // e.g. https://codeberg.org/api/v1/repos/privacyguides/privacyguides.org/languages + @Override + public String languages() throws IOException, InterruptedException, RsdResponseException { + return Utils.get(baseApiUrl + "/languages"); + } + + // e.g. https://codeberg.org/api/v1/repos/privacyguides/privacyguides.org/commits?limit=50 + // 50 is max limit, found out experimentally + // from https://codeberg.org/api/swagger#/repository/repoGetAllCommits we can see that paginating is 1-based + @Override + public CommitsPerWeek contributions() throws IOException, InterruptedException, RsdResponseException { + ZonedDateTime lastCommitHistoryTimestamp = commitsPerWeek.popLatestTimestamp(); + String baseUrl = baseApiUrl + "/commits?limit=50&page="; + boolean hasMorePages = true; + int page = 1; + + paginate: while (hasMorePages) { + HttpResponse response = Utils.getAsHttpResponse(baseUrl + page); + hasMorePages = "true".equals(response.headers().firstValue("x-hasmore").orElse("")); + ++page; + + JsonArray thisPageCommits = JsonParser.parseString(response.body()).getAsJsonArray(); + for (JsonElement element : thisPageCommits) { + JsonObject currentCommit = element.getAsJsonObject(); + String timeString = currentCommit.getAsJsonPrimitive("created").getAsString(); + ZonedDateTime time = ZonedDateTime.parse(timeString); + if (lastCommitHistoryTimestamp != null && time.isBefore(lastCommitHistoryTimestamp)) { + break paginate; + } + + commitsPerWeek.addCommits(time, 1); + } + } + + return commitsPerWeek; + } + + // e.g. https://codeberg.org/api/v1/repos/privacyguides/privacyguides.org/collaborators + // token required, unclear if you can see a repository's collaborators with your own token... + @Override + public Integer contributorCount() { + throw new UnsupportedOperationException(); + } + + static BasicGitData parseBasicData(String data) { + JsonObject jsonObject = JsonParser.parseString(data).getAsJsonObject(); + + Boolean archived = jsonObject.getAsJsonPrimitive("archived").getAsBoolean(); + Long starCount = jsonObject.getAsJsonPrimitive("stars_count").getAsLong(); + Integer forkCount = jsonObject.getAsJsonPrimitive("forks_count").getAsInt(); + Integer openIssueCount = jsonObject.getAsJsonPrimitive("open_issues_count").getAsInt(); + + return new BasicGitData(archived, null, starCount, forkCount, openIssueCount); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitData.java index dbe03ab..02020ba 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitData.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitData.java @@ -11,5 +11,4 @@ public record CommitData( BasicRepositoryData basicData, CommitsPerWeek commitHistory, ZonedDateTime commitHistoryScrapedAt -) { -} +) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeek.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeek.java index c8eae75..4398253 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeek.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeek.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2023 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2023 Netherlands eScience Center +// SPDX-FileCopyrightText: 2023 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center // SPDX-FileCopyrightText: 2025 Christian Meeßen (GFZ) // SPDX-FileCopyrightText: 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences // @@ -11,7 +11,6 @@ import com.google.gson.GsonBuilder; import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializer; - import java.time.DayOfWeek; import java.time.Instant; import java.time.Period; @@ -31,9 +30,12 @@ public class CommitsPerWeek { private final SortedMap data = new TreeMap<>(); static final Gson gson = new GsonBuilder() - .enableComplexMapKeySerialization() - .registerTypeAdapter(Instant.class, (JsonSerializer) (src, typeOfSrc, context) -> new JsonPrimitive(src.getEpochSecond())) - .create(); + .enableComplexMapKeySerialization() + .registerTypeAdapter( + Instant.class, + (JsonSerializer) (src, typeOfSrc, context) -> new JsonPrimitive(src.getEpochSecond()) + ) + .create(); SortedMap getData() { return new TreeMap<>(data); @@ -41,7 +43,10 @@ SortedMap getData() { public void addCommits(ZonedDateTime zonedDateTime, long count) { ZonedDateTime utcTime = zonedDateTime.withZoneSameInstant(ZoneOffset.UTC); - Instant sundayMidnight = utcTime.truncatedTo(ChronoUnit.DAYS).with(TemporalAdjusters.previousOrSame(DayOfWeek.SUNDAY)).toInstant(); + Instant sundayMidnight = utcTime + .truncatedTo(ChronoUnit.DAYS) + .with(TemporalAdjusters.previousOrSame(DayOfWeek.SUNDAY)) + .toInstant(); data.merge(sundayMidnight, count, Long::sum); } @@ -51,7 +56,7 @@ public void addCommits(Instant instant, long count) { } public ZonedDateTime popLatestTimestamp() { - if (this.data.size() > 0) { + if (!this.data.isEmpty()) { Instant lastTimeStamp = this.data.lastKey(); this.data.remove(lastTimeStamp); return ZonedDateTime.ofInstant(lastTimeStamp, ZoneId.systemDefault()); @@ -69,7 +74,11 @@ public void addMissingZeros() { Instant firstWeek = data.firstKey(); Instant lastWeek = data.lastKey(); - for (Instant currentWeek = firstWeek; currentWeek.isBefore(lastWeek); currentWeek = currentWeek.plus(Period.ofWeeks(1))) { + for ( + Instant currentWeek = firstWeek; + currentWeek.isBefore(lastWeek); + currentWeek = currentWeek.plus(Period.ofWeeks(1)) + ) { data.putIfAbsent(currentWeek, 0L); } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/ContributorDatabaseData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/ContributorDatabaseData.java index e0eb6bb..35774ab 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/ContributorDatabaseData.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/ContributorDatabaseData.java @@ -8,8 +8,7 @@ import java.time.ZonedDateTime; public record ContributorDatabaseData( - BasicRepositoryData basicData, - Integer contributorCount, - ZonedDateTime dataScrapedAt -) { -} + BasicRepositoryData basicData, + Integer contributorCount, + ZonedDateTime dataScrapedAt +) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/FourTuGitScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/FourTuGitScraper.java index 65ebe3b..4e1814e 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/FourTuGitScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/FourTuGitScraper.java @@ -1,20 +1,23 @@ -// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.git; import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.RsdResponseException; -import nl.esciencecenter.rsd.scraper.Utils; - import java.io.IOException; +import java.net.URI; import java.net.http.HttpResponse; +import java.time.Duration; import java.util.Objects; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; public class FourTuGitScraper implements GitScraper { + private final String repoUrl; + private static final Duration HTTP_TIMEOUT = Duration.ofSeconds(90); public FourTuGitScraper(String repoUrl) { this.repoUrl = Objects.requireNonNull(repoUrl); @@ -28,42 +31,67 @@ public BasicGitData basicData() throws IOException, InterruptedException, RsdRes // Example URL: https://data.4tu.nl/v3/datasets/454afe4e-08aa-4333-b6e8-7db7cebd0f83.git/languages @Override public String languages() throws IOException, InterruptedException, RsdResponseException { - String url = repoUrl + "/languages"; - HttpResponse response = Utils.getAsHttpResponse(url); + HttpResponse response = getRequest("/languages"); return switch (response.statusCode()) { - case 404 -> - throw new RsdResponseException(404, response.uri(), response.body(), "Not found, is the repository URL correct?"); + case 404 -> throw new RsdResponseException( + 404, + response.uri(), + response.body(), + "Not found, is the repository URL correct?" + ); case 200 -> response.body(); - default -> - throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response when downloading 4TU programming languages"); + default -> throw new RsdResponseException( + response.statusCode(), + response.uri(), + response.body(), + "Unexpected response when downloading 4TU programming languages" + ); }; } // Example URL: https://data.4tu.nl/v3/datasets/454afe4e-08aa-4333-b6e8-7db7cebd0f83.git/contributors @Override public CommitsPerWeek contributions() throws IOException, InterruptedException, RsdResponseException { - String url = repoUrl + "/contributors"; - HttpResponse response = Utils.getAsHttpResponse(url); + HttpResponse response = getRequest("/contributors"); return switch (response.statusCode()) { - case 404 -> - throw new RsdResponseException(404, response.uri(), response.body(), "Not found, is the repository URL correct?"); + case 404 -> throw new RsdResponseException( + 404, + response.uri(), + response.body(), + "Not found, is the repository URL correct?" + ); case 200 -> GithubScraper.parseCommits(response.body()); - default -> - throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response when downloading 4TU commit history"); + default -> throw new RsdResponseException( + response.statusCode(), + response.uri(), + response.body(), + "Unexpected response when downloading 4TU commit history" + ); }; } // Example URL: https://data.4tu.nl/v3/datasets/454afe4e-08aa-4333-b6e8-7db7cebd0f83.git/contributors @Override public Integer contributorCount() throws IOException, InterruptedException, RsdResponseException { - String url = repoUrl + "/contributors"; - HttpResponse response = Utils.getAsHttpResponse(url); + HttpResponse response = getRequest("/contributors"); return switch (response.statusCode()) { - case 404 -> - throw new RsdResponseException(404, response.uri(), response.body(), "Not found, is the repository URL correct?"); + case 404 -> throw new RsdResponseException( + 404, + response.uri(), + response.body(), + "Not found, is the repository URL correct?" + ); case 200 -> JsonParser.parseString(response.body()).getAsJsonArray().size(); - default -> - throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response when downloading 4TU commit history"); + default -> throw new RsdResponseException( + response.statusCode(), + response.uri(), + response.body(), + "Unexpected response when downloading 4TU commit history" + ); }; } + + private HttpResponse getRequest(String path) throws IOException, InterruptedException { + return Utils.getAsHttpResponse(URI.create(repoUrl + path), HTTP_TIMEOUT); + } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitScraper.java index 31dc31c..dd1783c 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitScraper.java @@ -6,11 +6,9 @@ package nl.esciencecenter.rsd.scraper.git; import java.io.IOException; - import nl.esciencecenter.rsd.scraper.RsdResponseException; public interface GitScraper { - BasicGitData basicData() throws IOException, InterruptedException, RsdResponseException; String languages() throws IOException, InterruptedException, RsdResponseException; diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GithubScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GithubScraper.java index 428c0f3..0e58581 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GithubScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GithubScraper.java @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2022 - 2024 Christian Meeßen (GFZ) -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) // SPDX-FileCopyrightText: 2022 - 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 @@ -11,12 +11,8 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.Config; -import nl.esciencecenter.rsd.scraper.RsdRateLimitException; -import nl.esciencecenter.rsd.scraper.RsdResponseException; -import nl.esciencecenter.rsd.scraper.Utils; - import java.io.IOException; +import java.net.URI; import java.net.http.HttpResponse; import java.time.Instant; import java.util.List; @@ -24,6 +20,10 @@ import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdRateLimitException; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; public class GithubScraper implements GitScraper { @@ -31,7 +31,9 @@ public class GithubScraper implements GitScraper { public final String organisation; public final String repo; private static final Pattern LINK_PATTERN = Pattern.compile("<([^>]+page=(\\d+)[^>]*)>; rel=\"([^\"]+)\""); - private static final Pattern GITHUB_URL_PATTERN = Pattern.compile("^https?://github\\.com/([^\\s/]+)/([^\\s/]+)/?$"); + private static final Pattern GITHUB_URL_PATTERN = Pattern.compile( + "^https?://github\\.com/([^\\s/]+)/([^\\s/]+)/?$" + ); private GithubScraper(String organisation, String repo) { this.organisation = Objects.requireNonNull(organisation); @@ -55,49 +57,79 @@ public static Optional create(String url) { /** * Returns the basic data of the repository. - * Example URL: https://api.github.com/repos/research-software-directory/RSD-as-a-service + * Example URL: https://api.github.com/repos/research-software-directory/RSD-as-a-service */ @Override public BasicGitData basicData() throws IOException, InterruptedException, RsdResponseException { Optional apiCredentials = Config.apiCredentialsGithub(); HttpResponse response; if (apiCredentials.isPresent()) { - response = Utils.getAsHttpResponse(BASE_API_URL + "/repos/" + organisation + "/" + repo, "Authorization", "Basic " + Utils.base64Encode(apiCredentials.get())); + response = Utils.getAsHttpResponse( + BASE_API_URL + "/repos/" + organisation + "/" + repo, + "Authorization", + "Basic " + Utils.base64Encode(apiCredentials.get()) + ); } else { response = Utils.getAsHttpResponse(BASE_API_URL + "/repos/" + organisation + "/" + repo); } return switch (response.statusCode()) { case 200 -> parseBasicData(response.body()); - case 404 -> - throw new RsdResponseException(404, response.uri(), response.body(), "Not found, is the repository URL correct?"); - case 403 -> - throw new RsdRateLimitException(403, response.uri(), response.body(), "Rate limit for GitHub probably reached"); - default -> - throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response"); + case 404 -> throw new RsdResponseException( + 404, + response.uri(), + response.body(), + "Not found, is the repository URL correct?" + ); + case 403 -> throw new RsdRateLimitException( + 403, + response.uri(), + response.body(), + "Rate limit for GitHub probably reached" + ); + default -> throw new RsdResponseException( + response.statusCode(), + response.uri(), + response.body(), + "Unexpected response" + ); }; } /** * Returns JSON as a String with the amount of lines written in each language. - * Example URL: https://api.github.com/repos/research-software-directory/RSD-as-a-service/languages + * Example URL: https://api.github.com/repos/research-software-directory/RSD-as-a-service/languages */ @Override public String languages() throws IOException, InterruptedException, RsdResponseException { - HttpResponse response = getAsHttpResponse(BASE_API_URL + "/repos/" + organisation + "/" + repo + "/languages"); + HttpResponse response = getAsHttpResponse( + BASE_API_URL + "/repos/" + organisation + "/" + repo + "/languages" + ); return switch (response.statusCode()) { - case 404 -> - throw new RsdResponseException(404, response.uri(), response.body(), "Not found, is the repository URL correct?"); - case 403 -> - throw new RsdRateLimitException(403, response.uri(), response.body(), "Rate limit for GitHub probably reached"); + case 404 -> throw new RsdResponseException( + 404, + response.uri(), + response.body(), + "Not found, is the repository URL correct?" + ); + case 403 -> throw new RsdRateLimitException( + 403, + response.uri(), + response.body(), + "Rate limit for GitHub probably reached" + ); case 200 -> response.body(); - default -> - throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response"); + default -> throw new RsdResponseException( + response.statusCode(), + response.uri(), + response.body(), + "Unexpected response" + ); }; } /** * Returns all contributors commit activity. - * https://docs.github.com/en/rest/reference/metrics#get-all-contributor-commit-activity= + * GitHub documetation. * Requesting commit activity requires a GitHub authentication token. *

* The returned string represents a JsonArray with one entry per contributor. THe information @@ -116,13 +148,15 @@ public String languages() throws IOException, InterruptedException, RsdResponseE * ] * } *

- * Example URL: https://api.github.com/repos/research-software-directory/RSD-as-a-service/stats/contributors + * Example URL: https://api.github.com/repos/research-software-directory/RSD-as-a-service/stats/contributors */ @Override public CommitsPerWeek contributions() throws IOException, InterruptedException, RsdResponseException { HttpResponse httpResponse = null; for (int i = 0; i < 2; i++) { - httpResponse = getAsHttpResponse(BASE_API_URL + "/repos/" + organisation + "/" + repo + "/stats/contributors"); + httpResponse = getAsHttpResponse( + BASE_API_URL + "/repos/" + organisation + "/" + repo + "/stats/contributors" + ); if (httpResponse.statusCode() != 202) break; Thread.sleep(3000L); @@ -130,12 +164,22 @@ public CommitsPerWeek contributions() throws IOException, InterruptedException, int status = httpResponse.statusCode(); if (status == 404) { - throw new RsdResponseException(status, httpResponse.uri(), httpResponse.body(), "Not found, is the repository URL correct?"); + throw new RsdResponseException( + status, + httpResponse.uri(), + httpResponse.body(), + "Not found, is the repository URL correct?" + ); } else if (status == 204) { // empty commit history return new CommitsPerWeek(); } else if (status == 403) { - throw new RsdRateLimitException(403, httpResponse.uri(), httpResponse.body(), "Rate limit for GitHub probably reached"); + throw new RsdRateLimitException( + 403, + httpResponse.uri(), + httpResponse.body(), + "Rate limit for GitHub probably reached" + ); } else if (status == 202) { // response not ready yet return null; @@ -152,13 +196,25 @@ public CommitsPerWeek contributions() throws IOException, InterruptedException, public Integer contributorCount() throws IOException, InterruptedException, RsdResponseException { // we request one contributor per page and just extract the number of pages from the headers // see https://docs.github.com/en/rest/guides/using-pagination-in-the-rest-api?apiVersion=2022-11-28 - HttpResponse httpResponse = getAsHttpResponse(BASE_API_URL + "/repos/" + organisation + "/" + repo + "/contributors?per_page=1"); + HttpResponse httpResponse = getAsHttpResponse( + BASE_API_URL + "/repos/" + organisation + "/" + repo + "/contributors?per_page=1" + ); int status = httpResponse.statusCode(); if (status == 404) { - throw new RsdResponseException(404, httpResponse.uri(), httpResponse.body(), "Not found, is the repository URL correct?"); + throw new RsdResponseException( + 404, + httpResponse.uri(), + httpResponse.body(), + "Not found, is the repository URL correct?" + ); } else if (status == 403) { - throw new RsdRateLimitException(403, httpResponse.uri(), httpResponse.body(), "Rate limit for GitHub probably reached"); + throw new RsdRateLimitException( + 403, + httpResponse.uri(), + httpResponse.body(), + "Rate limit for GitHub probably reached" + ); } else if (status != 200) { throw new RsdResponseException(status, httpResponse.uri(), httpResponse.body(), "Unexpected response"); } else { @@ -174,6 +230,40 @@ public Integer contributorCount() throws IOException, InterruptedException, RsdR } } + /** + * + * Retrieve the URL of the raw README file. + * Example URL: https://api.github.com/repos/research-software-directory/RSD-as-a-service/readme. + * + * @return the URL of the raw README of the repo if present + */ + public Optional rawReadmeUrl() throws IOException, InterruptedException, RsdResponseException { + HttpResponse httpResponse = getAsHttpResponse( + BASE_API_URL + "/repos/" + organisation + "/" + repo + "/readme" + ); + + int status = httpResponse.statusCode(); + if (status == 404) { + throw new RsdResponseException( + 404, + httpResponse.uri(), + httpResponse.body(), + "Not found, is the repository URL correct?" + ); + } else if (status == 403) { + throw new RsdRateLimitException( + 403, + httpResponse.uri(), + httpResponse.body(), + "Rate limit for GitHub probably reached" + ); + } else if (status != 200) { + throw new RsdResponseException(status, httpResponse.uri(), httpResponse.body(), "Unexpected response"); + } else { + return parseReadmeUrl(httpResponse.body()); + } + } + static CommitsPerWeek parseCommits(String json) { CommitsPerWeek commits = new CommitsPerWeek(); JsonArray commitsPerContributor = JsonParser.parseString(json).getAsJsonArray(); @@ -198,20 +288,14 @@ static BasicGitData parseBasicData(String json) { Boolean archived = jsonObject.getAsJsonPrimitive("archived").getAsBoolean(); JsonElement jsonLicense = jsonObject.get("license"); - String license = jsonLicense.isJsonNull() ? null : jsonLicense.getAsJsonObject() - .getAsJsonPrimitive("spdx_id") - .getAsString(); + String license = jsonLicense.isJsonNull() + ? null + : jsonLicense.getAsJsonObject().getAsJsonPrimitive("spdx_id").getAsString(); Long starCount = jsonObject.getAsJsonPrimitive("stargazers_count").getAsLong(); Integer forkCount = jsonObject.getAsJsonPrimitive("forks_count").getAsInt(); Integer openIssueCount = jsonObject.getAsJsonPrimitive("open_issues_count").getAsInt(); - return new BasicGitData( - archived, - license, - starCount, - forkCount, - openIssueCount - ); + return new BasicGitData(archived, license, starCount, forkCount, openIssueCount); } // return an object with the URL of the last page and the number of the last page respectively @@ -222,7 +306,7 @@ static String[] lastPageFromLinkHeader(List links) { Matcher matcher = LINK_PATTERN.matcher(link); while (matcher.find()) { String relation = matcher.group(3); - if (relation.equals("last")) return new String[]{matcher.group(1), matcher.group(2)}; + if (relation.equals("last")) return new String[] { matcher.group(1), matcher.group(2) }; } } @@ -237,4 +321,19 @@ static HttpResponse getAsHttpResponse(String url) throws IOException, In return Utils.getAsHttpResponse(url); } } + + static Optional parseReadmeUrl(String json) { + JsonObject root = JsonParser.parseString(json).getAsJsonObject(); + JsonElement rawReadmeUrlJson = root.get("download_url"); + + if ( + rawReadmeUrlJson == null || + !rawReadmeUrlJson.isJsonPrimitive() || + !rawReadmeUrlJson.getAsJsonPrimitive().isString() + ) { + return Optional.empty(); + } + + return Optional.of(URI.create(rawReadmeUrlJson.getAsString())); + } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitlabScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitlabScraper.java index 479a090..aec13c1 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitlabScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/GitlabScraper.java @@ -1,13 +1,17 @@ -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center // SPDX-FileCopyrightText: 2022 - 2025 Christian Meeßen (GFZ) // SPDX-FileCopyrightText: 2022 - 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2022 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2026 Netherlands eScience Center // SPDX-FileCopyrightText: 2025 Paula Stock (GFZ) // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.git; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import java.io.IOException; import java.net.URI; import java.net.http.HttpClient; @@ -15,12 +19,7 @@ import java.net.http.HttpResponse; import java.time.Duration; import java.time.ZonedDateTime; - -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; - +import java.util.Optional; import nl.esciencecenter.rsd.scraper.RsdRateLimitException; import nl.esciencecenter.rsd.scraper.RsdResponseException; import nl.esciencecenter.rsd.scraper.Utils; @@ -40,15 +39,24 @@ public class GitlabScraper implements GitScraper { * @param projectPath The full path to the project */ public GitlabScraper(String gitLabApiUrl, String projectPath) { - this.projectPath = projectPath.endsWith(".git") ? projectPath.substring(0, projectPath.length() - 4) : projectPath; + this.projectPath = projectPath.endsWith(".git") + ? projectPath.substring(0, projectPath.length() - 4) + : projectPath; this.apiUri = gitLabApiUrl + "/v4"; this.graphqlUri = gitLabApiUrl + "/graphql"; this.lastCommitHistoryTimestamp = null; this.commitsPerWeek = new CommitsPerWeek(); } - public GitlabScraper(String gitLabApiUrl, String projectPath, ZonedDateTime lastCommitHistoryTimestamp, CommitsPerWeek existingCommitsPerWeek) { - this.projectPath = projectPath.endsWith(".git") ? projectPath.substring(0, projectPath.length() - 4) : projectPath; + public GitlabScraper( + String gitLabApiUrl, + String projectPath, + ZonedDateTime lastCommitHistoryTimestamp, + CommitsPerWeek existingCommitsPerWeek + ) { + this.projectPath = projectPath.endsWith(".git") + ? projectPath.substring(0, projectPath.length() - 4) + : projectPath; this.apiUri = gitLabApiUrl + "/v4"; this.graphqlUri = gitLabApiUrl + "/graphql"; this.lastCommitHistoryTimestamp = lastCommitHistoryTimestamp; @@ -98,26 +106,43 @@ public String languages() throws IOException, InterruptedException, RsdResponseE */ @Override public CommitsPerWeek contributions() throws IOException, InterruptedException, RsdResponseException { - String since=""; + String since = ""; if (lastCommitHistoryTimestamp != null) { since = "&since=" + lastCommitHistoryTimestamp.toString(); } String page = "1"; boolean done = false; while (!done) { - HttpRequest request = HttpRequest.newBuilder().GET() - .uri(URI.create(apiUri + "/projects/" + Utils.urlEncode(projectPath) - + "/repository/commits?per_page=100&order=default&page=" + page + since)) + HttpRequest request = HttpRequest.newBuilder() + .GET() + .uri( + URI.create( + apiUri + + "/projects/" + + Utils.urlEncode(projectPath) + + "/repository/commits?per_page=100&order=default&page=" + + page + + since + ) + ) .timeout(Duration.ofSeconds(30)) .build(); HttpResponse response; try (HttpClient client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build()) { response = client.send(request, HttpResponse.BodyHandlers.ofString()); } - if (response.statusCode() == 429) - throw new RsdRateLimitException(429, response.uri(), response.body(), "API rate limit reached for GitLab"); - if (response.statusCode() == 404) - throw new RsdResponseException(404, response.uri(), response.body(), "Not found, is the repository URL correct?"); + if (response.statusCode() == 429) throw new RsdRateLimitException( + 429, + response.uri(), + response.body(), + "API rate limit reached for GitLab" + ); + if (response.statusCode() == 404) throw new RsdResponseException( + 404, + response.uri(), + response.body(), + "Not found, is the repository URL correct?" + ); String body = response.body(); parseCommitPage(body, commitsPerWeek); @@ -131,18 +156,60 @@ public CommitsPerWeek contributions() throws IOException, InterruptedException, // Example URL: https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab-shell/repository/contributors @Override public Integer contributorCount() throws IOException, InterruptedException, RsdResponseException { - HttpResponse httpResponse = Utils.getAsHttpResponse(apiUri + "/projects/" + Utils.urlEncode(projectPath) + "/repository/contributors"); + HttpResponse httpResponse = Utils.getAsHttpResponse( + apiUri + "/projects/" + Utils.urlEncode(projectPath) + "/repository/contributors" + ); - if (httpResponse.statusCode() == 429) - throw new RsdRateLimitException(429, httpResponse.uri(), httpResponse.body(), "Rate limit reached for GitLab"); - if (httpResponse.statusCode() == 404) - throw new RsdResponseException(404, httpResponse.uri(), httpResponse.body(), "Not found, is the repository URL correct?"); + if (httpResponse.statusCode() == 429) throw new RsdRateLimitException( + 429, + httpResponse.uri(), + httpResponse.body(), + "Rate limit reached for GitLab" + ); + if (httpResponse.statusCode() == 404) throw new RsdResponseException( + 404, + httpResponse.uri(), + httpResponse.body(), + "Not found, is the repository URL correct?" + ); // see https://docs.gitlab.com/ee/api/rest/index.html#other-pagination-headers String totalItemsHeader = httpResponse.headers().firstValue("x-total").orElseThrow(); return Integer.parseInt(totalItemsHeader); } + /** + * + * Retrieve the URL of the raw README file. + * Example URL: https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab-shell. + * + * @return the URL of the raw README of the repo if present + */ + public Optional rawReadmeUrl() throws IOException, InterruptedException, RsdResponseException { + HttpResponse httpResponse = Utils.getAsHttpResponse( + apiUri + "/projects/" + Utils.urlEncode(projectPath) + ); + + int status = httpResponse.statusCode(); + if (httpResponse.statusCode() == 429) throw new RsdRateLimitException( + 429, + httpResponse.uri(), + httpResponse.body(), + "Rate limit reached for GitLab" + ); + if (httpResponse.statusCode() == 404) throw new RsdResponseException( + 404, + httpResponse.uri(), + httpResponse.body(), + "Not found, is the repository URL correct?" + ); + if (status != 200) { + throw new RsdResponseException(status, httpResponse.uri(), httpResponse.body(), "Unexpected response"); + } else { + return parseReadmeUrl(httpResponse.body()); + } + } + static void parseCommitPage(String json, CommitsPerWeek commitsToFill) { JsonArray thisPageCommits = JsonParser.parseString(json).getAsJsonArray(); @@ -155,21 +222,43 @@ static void parseCommitPage(String json, CommitsPerWeek commitsToFill) { } } - private Boolean checkIfArchived() throws IOException, InterruptedException, RsdResponseException { - String graphqlQuery = String.format(""" - { - project(fullPath: "%s/%s") { - archived - } + private Boolean checkIfArchived() { + String graphqlQuery = String.format( + """ + { + project(fullPath: "%s/%s") { + archived } - """, - projectPath.substring(0, projectPath.indexOf("/")), - projectPath.substring(projectPath.indexOf("/") + 1)); + } + """, + projectPath.substring(0, projectPath.indexOf("/")), + projectPath.substring(projectPath.indexOf("/") + 1) + ); JsonObject body = new JsonObject(); body.addProperty("query", graphqlQuery); String response = Utils.post(graphqlUri, body.toString(), "Content-Type", "application/json"); + return parseArchivedResponse(response); + } + + static Boolean parseArchivedResponse(String response) { JsonObject jsonObject = JsonParser.parseString(response).getAsJsonObject(); - return jsonObject.getAsJsonObject("data").getAsJsonObject("project").getAsJsonPrimitive("archived").getAsBoolean(); + + JsonElement dataElement = jsonObject.get("data"); + if (dataElement == null || !dataElement.isJsonObject()) { + return null; + } + + JsonElement projectElement = dataElement.getAsJsonObject().get("project"); + if (projectElement == null || !projectElement.isJsonObject()) { + return null; + } + + JsonElement archivedElement = projectElement.getAsJsonObject().get("archived"); + if (archivedElement == null || !archivedElement.isJsonPrimitive()) { + return null; + } + + return archivedElement.getAsJsonPrimitive().isBoolean() ? archivedElement.getAsBoolean() : null; } static BasicGitData parseBasicData(String json, Boolean archived) { @@ -180,13 +269,30 @@ static BasicGitData parseBasicData(String json, Boolean archived) { Long starCount = jsonObject.getAsJsonPrimitive("star_count").getAsLong(); Integer forkCount = jsonObject.getAsJsonPrimitive("forks_count").getAsInt(); - return new BasicGitData( - archived, - license, - starCount, - forkCount, - null - ); + return new BasicGitData(archived, license, starCount, forkCount, null); } + /** + * + * This parses the JSON response and transforms the returned "HTML" URL to a "raw" URL. + * + * @param json the response JSON to parse + * @return the raw README URL + */ + static Optional parseReadmeUrl(String json) { + JsonObject root = JsonParser.parseString(json).getAsJsonObject(); + JsonElement htmlReadmeJson = root.get("readme_url"); + + if ( + htmlReadmeJson == null || + !htmlReadmeJson.isJsonPrimitive() || + !htmlReadmeJson.getAsJsonPrimitive().isString() + ) { + return Optional.empty(); + } + + String htmlReadmeUrl = htmlReadmeJson.getAsString(); + String rawReadmeUrl = htmlReadmeUrl.replace("/-/blob/", "/-/raw/"); + return Optional.of(URI.create(rawReadmeUrl)); + } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/LanguagesData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/LanguagesData.java index 578ffa9..2db025c 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/LanguagesData.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/LanguagesData.java @@ -7,9 +7,4 @@ import java.time.ZonedDateTime; -public record LanguagesData( - BasicRepositoryData basicData, - String languages, - ZonedDateTime languagesScrapedAt -) { -} +public record LanguagesData(BasicRepositoryData basicData, String languages, ZonedDateTime languagesScrapedAt) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainBasicData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainBasicData.java index c2c3be5..ede28c2 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainBasicData.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainBasicData.java @@ -1,10 +1,16 @@ -// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2023 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2025 Dusan Mijatovic (Netherlands eScience Center) // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.git; +import java.net.URI; +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; import nl.esciencecenter.rsd.scraper.Config; import nl.esciencecenter.rsd.scraper.RsdRateLimitException; import nl.esciencecenter.rsd.scraper.RsdResponseException; @@ -12,12 +18,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.URI; -import java.time.ZonedDateTime; -import java.util.Collection; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; - public class MainBasicData { private static final Logger LOGGER = LoggerFactory.getLogger(MainBasicData.class); @@ -29,6 +29,7 @@ public static void main(String[] args) { scrapeGitHub(); scrapeGitLab(); + scrapeCodeberg(); long time = System.currentTimeMillis() - t1; @@ -36,7 +37,10 @@ public static void main(String[] args) { } private static void scrapeGitHub() { - PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITHUB); + PostgrestConnector softwareInfoRepository = new PostgrestConnector( + Config.backendBaseUrl() + "/repository_url", + CodePlatformProvider.GITHUB + ); Collection dataToScrape = softwareInfoRepository.statsData(Config.maxRequestsGithub()); CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; ZonedDateTime scrapedAt = ZonedDateTime.now(); @@ -48,7 +52,15 @@ private static void scrapeGitHub() { Optional githubScraperOptional = GithubScraper.create(repoUrl); if (githubScraperOptional.isEmpty()) { - Utils.saveErrorMessageInDatabase("Not a valid GitHub URL: " + repoUrl, "repository_url", "basic_data_last_error", basicData.software().toString(), "software", scrapedAt, "basic_data_scraped_at"); + Utils.saveErrorMessageInDatabase( + "Not a valid GitHub URL: " + repoUrl, + "repository_url", + "basic_data_last_error", + basicData.software().toString(), + "id", + scrapedAt, + "basic_data_scraped_at" + ); return; } @@ -57,14 +69,53 @@ private static void scrapeGitHub() { BasicGitDatabaseData updatedData = new BasicGitDatabaseData(basicData, scrapedBasicData, scrapedAt); softwareInfoRepository.saveBasicData(updatedData); } catch (RsdRateLimitException e) { - Utils.saveExceptionInDatabase("GitHub basic data scraper", "repository_url", basicData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "basic_data_last_error", basicData.software().toString(), "software", null, null); + Utils.saveExceptionInDatabase( + "GitHub basic data scraper", + "repository_url", + basicData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "basic_data_last_error", + basicData.software().toString(), + "id", + null, + null + ); } catch (RsdResponseException e) { - Utils.saveExceptionInDatabase("GitHub basic data scraper", "repository_url", basicData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "basic_data_last_error", basicData.software().toString(), "software", scrapedAt, "basic_data_scraped_at"); + Utils.saveExceptionInDatabase( + "GitHub basic data scraper", + "repository_url", + basicData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "basic_data_last_error", + basicData.software().toString(), + "id", + scrapedAt, + "basic_data_scraped_at" + ); } catch (Exception e) { - Utils.saveExceptionInDatabase("GitHub basic data scraper", "repository_url", basicData.software(), e); - Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "basic_data_last_error", basicData.software().toString(), "software", scrapedAt, "basic_data_scraped_at"); + Utils.saveExceptionInDatabase( + "GitHub basic data scraper", + "repository_url", + basicData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "repository_url", + "basic_data_last_error", + basicData.software().toString(), + "id", + scrapedAt, + "basic_data_scraped_at" + ); } }); futures[i] = future; @@ -74,8 +125,11 @@ private static void scrapeGitHub() { } private static void scrapeGitLab() { - PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITLAB); - Collection dataToScrape = softwareInfoRepository.statsData(Config.maxRequestsGithub()); + PostgrestConnector softwareInfoRepository = new PostgrestConnector( + Config.backendBaseUrl() + "/repository_url", + CodePlatformProvider.GITLAB + ); + Collection dataToScrape = softwareInfoRepository.statsData(Config.maxRequestsGitLab()); CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; ZonedDateTime scrapedAt = ZonedDateTime.now(); int i = 0; @@ -92,14 +146,141 @@ private static void scrapeGitLab() { BasicGitDatabaseData updatedData = new BasicGitDatabaseData(basicData, scrapedBasicData, scrapedAt); softwareInfoRepository.saveBasicData(updatedData); } catch (RsdRateLimitException e) { - Utils.saveExceptionInDatabase("GitLab basic data scraper", "repository_url", basicData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "basic_data_last_error", basicData.software().toString(), "software", null, null); + Utils.saveExceptionInDatabase( + "GitLab basic data scraper", + "repository_url", + basicData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "basic_data_last_error", + basicData.software().toString(), + "id", + null, + null + ); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase( + "GitLab basic data scraper", + "repository_url", + basicData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "basic_data_last_error", + basicData.software().toString(), + "id", + scrapedAt, + "basic_data_scraped_at" + ); + } catch (Exception e) { + Utils.saveExceptionInDatabase( + "GitLab basic data scraper", + "repository_url", + basicData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "repository_url", + "basic_data_last_error", + basicData.software().toString(), + "id", + scrapedAt, + "basic_data_scraped_at" + ); + } + }); + futures[i] = future; + i++; + } + CompletableFuture.allOf(futures).join(); + } + + private static void scrapeCodeberg() { + PostgrestConnector softwareInfoRepository = new PostgrestConnector( + Config.backendBaseUrl() + "/repository_url", + CodePlatformProvider.CODEBERG + ); + Collection dataToScrape = softwareInfoRepository.statsData(Config.maxRequestsCodeberg()); + CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; + ZonedDateTime scrapedAt = ZonedDateTime.now(); + int i = 0; + for (BasicRepositoryData basicData : dataToScrape) { + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + String repoUrl = basicData.url(); + + Optional codebergScraperOptional = CodebergScraper.fromUrl(repoUrl); + if (codebergScraperOptional.isEmpty()) { + Utils.saveErrorMessageInDatabase( + "Not a valid Codeberg URL: " + repoUrl, + "repository_url", + "basic_data_last_error", + basicData.software().toString(), + "id", + scrapedAt, + "basic_data_scraped_at" + ); + return; + } + + CodebergScraper codebergScraper = codebergScraperOptional.get(); + BasicGitData scrapedBasicData = codebergScraper.basicData(); + BasicGitDatabaseData updatedData = new BasicGitDatabaseData(basicData, scrapedBasicData, scrapedAt); + softwareInfoRepository.saveBasicData(updatedData); + } catch (RsdRateLimitException e) { + Utils.saveExceptionInDatabase( + "Codeberg basic data scraper", + "repository_url", + basicData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "basic_data_last_error", + basicData.software().toString(), + "id", + null, + null + ); } catch (RsdResponseException e) { - Utils.saveExceptionInDatabase("GitLab basic data scraper", "repository_url", basicData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "basic_data_last_error", basicData.software().toString(), "software", scrapedAt, "basic_data_scraped_at"); + Utils.saveExceptionInDatabase( + "Codeberg basic data scraper", + "repository_url", + basicData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "basic_data_last_error", + basicData.software().toString(), + "id", + scrapedAt, + "basic_data_scraped_at" + ); } catch (Exception e) { - Utils.saveExceptionInDatabase("GitLab basic data scraper", "repository_url", basicData.software(), e); - Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "basic_data_last_error", basicData.software().toString(), "software", scrapedAt, "basic_data_scraped_at"); + Utils.saveExceptionInDatabase( + "Codeberg basic data scraper", + "repository_url", + basicData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "repository_url", + "basic_data_last_error", + basicData.software().toString(), + "id", + scrapedAt, + "basic_data_scraped_at" + ); } }); futures[i] = future; diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainCommits.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainCommits.java index 16fcd4a..f6447f7 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainCommits.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainCommits.java @@ -1,12 +1,18 @@ -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center // SPDX-FileCopyrightText: 2022 - 2025 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2022 - 2025 Ewan Cahen (Netherlands eScience Center) // SPDX-FileCopyrightText: 2022 - 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2022 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2025 Dusan Mijatovic (Netherlands eScience Center) // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.git; +import java.net.URI; +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; import nl.esciencecenter.rsd.scraper.Config; import nl.esciencecenter.rsd.scraper.RsdRateLimitException; import nl.esciencecenter.rsd.scraper.RsdResponseException; @@ -14,18 +20,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.URI; -import java.time.ZonedDateTime; -import java.util.Collection; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; - public class MainCommits { private static final Logger LOGGER = LoggerFactory.getLogger(MainCommits.class); public static void main(String[] args) { - LOGGER.info("Start scraping commits"); long t1 = System.currentTimeMillis(); @@ -33,6 +32,7 @@ public static void main(String[] args) { scrapeGitHub(); scrapeGitLab(); scrape4tu(); + scrapeCodeberg(); long time = System.currentTimeMillis() - t1; @@ -40,8 +40,13 @@ public static void main(String[] args) { } private static void scrapeGitLab() { - PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITLAB); - Collection dataToScrape = softwareInfoRepository.commitDataWithHistory(Config.maxRequestsGitLab()); + PostgrestConnector softwareInfoRepository = new PostgrestConnector( + Config.backendBaseUrl() + "/repository_url", + CodePlatformProvider.GITLAB + ); + Collection dataToScrape = softwareInfoRepository.commitDataWithHistory( + Config.maxRequestsGitLab() + ); CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; ZonedDateTime scrapedAt = ZonedDateTime.now(); int i = 0; @@ -56,20 +61,67 @@ private static void scrapeGitLab() { CommitsPerWeek existingCommitsPerWeek = repositoryDataToScrape.commitsPerWeek(); ZonedDateTime lastCommitHistoryTimestamp = existingCommitsPerWeek.popLatestTimestamp(); - CommitsPerWeek scrapedCommits = new GitlabScraper(apiUrl, projectPath, lastCommitHistoryTimestamp, existingCommitsPerWeek).contributions(); + CommitsPerWeek scrapedCommits = new GitlabScraper( + apiUrl, + projectPath, + lastCommitHistoryTimestamp, + existingCommitsPerWeek + ).contributions(); - BasicRepositoryData basicData = new BasicRepositoryData(repositoryDataToScrape.software(), repositoryDataToScrape.url()); + BasicRepositoryData basicData = new BasicRepositoryData( + repositoryDataToScrape.software(), + repositoryDataToScrape.url() + ); CommitData updatedData = new CommitData(basicData, scrapedCommits, scrapedAt); softwareInfoRepository.saveCommitData(updatedData); } catch (RsdRateLimitException e) { - Utils.saveExceptionInDatabase("GitLab commit scraper", "repository_url", repositoryDataToScrape.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "commit_history_last_error", repositoryDataToScrape.software().toString(), "software", null, null); + Utils.saveExceptionInDatabase( + "GitLab commit scraper", + "repository_url", + repositoryDataToScrape.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "commit_history_last_error", + repositoryDataToScrape.software().toString(), + "id", + null, + null + ); } catch (RsdResponseException e) { - Utils.saveExceptionInDatabase("GitLab commit scraper", "repository_url", repositoryDataToScrape.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "commit_history_last_error", repositoryDataToScrape.software().toString(), "software", scrapedAt, "commit_history_scraped_at"); + Utils.saveExceptionInDatabase( + "GitLab commit scraper", + "repository_url", + repositoryDataToScrape.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "commit_history_last_error", + repositoryDataToScrape.software().toString(), + "id", + scrapedAt, + "commit_history_scraped_at" + ); } catch (Exception e) { - Utils.saveExceptionInDatabase("GitLab commit scraper", "repository_url", repositoryDataToScrape.software(), e); - Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "commit_history_last_error", repositoryDataToScrape.software().toString(), "software", scrapedAt, "commit_history_scraped_at"); + Utils.saveExceptionInDatabase( + "GitLab commit scraper", + "repository_url", + repositoryDataToScrape.software(), + e + ); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "repository_url", + "commit_history_last_error", + repositoryDataToScrape.software().toString(), + "id", + scrapedAt, + "commit_history_scraped_at" + ); } }); futures[i] = future; @@ -79,7 +131,10 @@ private static void scrapeGitLab() { } private static void scrapeGitHub() { - PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITHUB); + PostgrestConnector softwareInfoRepository = new PostgrestConnector( + Config.backendBaseUrl() + "/repository_url", + CodePlatformProvider.GITHUB + ); Collection dataToScrape = softwareInfoRepository.commitData(Config.maxRequestsGithub()); CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; ZonedDateTime scrapedAt = ZonedDateTime.now(); @@ -91,7 +146,15 @@ private static void scrapeGitHub() { Optional githubScraperOptional = GithubScraper.create(repoUrl); if (githubScraperOptional.isEmpty()) { - Utils.saveErrorMessageInDatabase("Not a valid GitHub URL: " + repoUrl, "repository_url", "commit_history_last_error", commitData.software().toString(), "software", scrapedAt, "commit_history_scraped_at"); + Utils.saveErrorMessageInDatabase( + "Not a valid GitHub URL: " + repoUrl, + "repository_url", + "commit_history_last_error", + commitData.software().toString(), + "id", + scrapedAt, + "commit_history_scraped_at" + ); return; } @@ -102,13 +165,37 @@ private static void scrapeGitHub() { } catch (RsdRateLimitException e) { // in case we hit the rate limit, we don't update the scraped_at time, so it gets scraped first next time Utils.saveExceptionInDatabase("GitHub commit scraper", "repository_url", commitData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "commit_history_last_error", commitData.software().toString(), "software", null, null); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "commit_history_last_error", + commitData.software().toString(), + "id", + null, + null + ); } catch (RsdResponseException e) { Utils.saveExceptionInDatabase("GitHub commit scraper", "repository_url", commitData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "commit_history_last_error", commitData.software().toString(), "software", scrapedAt, "commit_history_scraped_at"); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "commit_history_last_error", + commitData.software().toString(), + "id", + scrapedAt, + "commit_history_scraped_at" + ); } catch (Exception e) { Utils.saveExceptionInDatabase("GitHub commit scraper", "repository_url", commitData.software(), e); - Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "commit_history_last_error", commitData.software().toString(), "software", scrapedAt, "commit_history_scraped_at"); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "repository_url", + "commit_history_last_error", + commitData.software().toString(), + "id", + scrapedAt, + "commit_history_scraped_at" + ); } }); futures[i] = future; @@ -118,8 +205,11 @@ private static void scrapeGitHub() { } private static void scrape4tu() { - PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.FOURTU); - Collection dataToScrape = softwareInfoRepository.commitData(Config.maxRequestsGithub()); + PostgrestConnector softwareInfoRepository = new PostgrestConnector( + Config.backendBaseUrl() + "/repository_url", + CodePlatformProvider.FOURTU + ); + Collection dataToScrape = softwareInfoRepository.commitData(Config.maxRequests4tu()); CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; ZonedDateTime scrapedAt = ZonedDateTime.now(); int i = 0; @@ -135,13 +225,133 @@ private static void scrape4tu() { } catch (RsdRateLimitException e) { // in case we hit the rate limit, we don't update the scraped_at time, so it gets scraped first next time Utils.saveExceptionInDatabase("4TU commit scraper", "repository_url", commitData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "commit_history_last_error", commitData.software().toString(), "software", null, null); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "commit_history_last_error", + commitData.software().toString(), + "id", + null, + null + ); } catch (RsdResponseException e) { Utils.saveExceptionInDatabase("4TU commit scraper", "repository_url", commitData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "commit_history_last_error", commitData.software().toString(), "software", scrapedAt, "commit_history_scraped_at"); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "commit_history_last_error", + commitData.software().toString(), + "id", + scrapedAt, + "commit_history_scraped_at" + ); } catch (Exception e) { Utils.saveExceptionInDatabase("4TU commit scraper", "repository_url", commitData.software(), e); - Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "commit_history_last_error", commitData.software().toString(), "software", scrapedAt, "commit_history_scraped_at"); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "repository_url", + "commit_history_last_error", + commitData.software().toString(), + "id", + scrapedAt, + "commit_history_scraped_at" + ); + } + }); + futures[i] = future; + i++; + } + CompletableFuture.allOf(futures).join(); + } + + private static void scrapeCodeberg() { + PostgrestConnector softwareInfoRepository = new PostgrestConnector( + Config.backendBaseUrl() + "/repository_url", + CodePlatformProvider.CODEBERG + ); + Collection dataToScrape = softwareInfoRepository.commitDataWithHistory( + Config.maxRequestsCodeberg() + ); + CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; + ZonedDateTime scrapedAt = ZonedDateTime.now(); + int i = 0; + for (BasicRepositoryDataWithHistory commitData : dataToScrape) { + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + softwareInfoRepository.saveCommitHistoryScrapedAt(commitData.software(), scrapedAt); + String repoUrl = commitData.url(); + + Optional codebergScraperOptional = CodebergScraper.fromUrl( + repoUrl, + commitData.commitsPerWeek() + ); + if (codebergScraperOptional.isEmpty()) { + Utils.saveErrorMessageInDatabase( + "Not a valid Codeberg URL: " + repoUrl, + "repository_url", + "commit_history_last_error", + commitData.software().toString(), + "id", + scrapedAt, + "commit_history_scraped_at" + ); + return; + } + + CodebergScraper codebergScraper = codebergScraperOptional.get(); + CommitsPerWeek scrapedCommits = codebergScraper.contributions(); + BasicRepositoryData basicData = new BasicRepositoryData(commitData.software(), commitData.url()); + CommitData updatedData = new CommitData(basicData, scrapedCommits, scrapedAt); + softwareInfoRepository.saveCommitData(updatedData); + } catch (RsdRateLimitException e) { + // in case we hit the rate limit, we don't update the scraped_at time, so it gets scraped first next time + Utils.saveExceptionInDatabase( + "Codeberg commit scraper", + "repository_url", + commitData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "commit_history_last_error", + commitData.software().toString(), + "id", + null, + null + ); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase( + "Codeberg commit scraper", + "repository_url", + commitData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "commit_history_last_error", + commitData.software().toString(), + "id", + scrapedAt, + "commit_history_scraped_at" + ); + } catch (Exception e) { + Utils.saveExceptionInDatabase( + "Codeberg commit scraper", + "repository_url", + commitData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "repository_url", + "commit_history_last_error", + commitData.software().toString(), + "id", + scrapedAt, + "commit_history_scraped_at" + ); } }); futures[i] = future; diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainContributors.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainContributors.java index e2dd0dd..e2e20e8 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainContributors.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainContributors.java @@ -1,10 +1,16 @@ -// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2023 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2025 Dusan Mijatovic (Netherlands eScience Center) // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.git; +import java.net.URI; +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; import nl.esciencecenter.rsd.scraper.Config; import nl.esciencecenter.rsd.scraper.RsdRateLimitException; import nl.esciencecenter.rsd.scraper.RsdResponseException; @@ -12,12 +18,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.URI; -import java.time.ZonedDateTime; -import java.util.Collection; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; - public class MainContributors { private static final Logger LOGGER = LoggerFactory.getLogger(MainContributors.class); @@ -37,8 +37,13 @@ public static void main(String[] args) { } private static void scrapeGitHub() { - PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITHUB); - Collection dataToScrape = softwareInfoRepository.contributorData(Config.maxRequestsGithub()); + PostgrestConnector softwareInfoRepository = new PostgrestConnector( + Config.backendBaseUrl() + "/repository_url", + CodePlatformProvider.GITHUB + ); + Collection dataToScrape = softwareInfoRepository.contributorData( + Config.maxRequestsGithub() + ); CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; ZonedDateTime scrapedAt = ZonedDateTime.now(); int i = 0; @@ -49,28 +54,75 @@ private static void scrapeGitHub() { Optional githubScraperOptional = GithubScraper.create(repoUrl); if (githubScraperOptional.isEmpty()) { - Utils.saveErrorMessageInDatabase("Not a valid GitHub URL: " + repoUrl, "repository_url", "contributor_count_last_error", contributorData.software() - .toString(), "software", scrapedAt, "contributor_count_scraped_at"); + Utils.saveErrorMessageInDatabase( + "Not a valid GitHub URL: " + repoUrl, + "repository_url", + "contributor_count_last_error", + contributorData.software().toString(), + "id", + scrapedAt, + "contributor_count_scraped_at" + ); return; } GithubScraper githubScraper = githubScraperOptional.get(); Integer scrapedContributorData = githubScraper.contributorCount(); - ContributorDatabaseData updatedData = new ContributorDatabaseData(new BasicRepositoryData(contributorData.software(), null), scrapedContributorData, scrapedAt); + ContributorDatabaseData updatedData = new ContributorDatabaseData( + new BasicRepositoryData(contributorData.software(), null), + scrapedContributorData, + scrapedAt + ); softwareInfoRepository.saveContributorCount(updatedData); } catch (RsdRateLimitException e) { // in case we hit the rate limit, we don't update the scraped_at time, so it gets scraped first next time - Utils.saveExceptionInDatabase("GitHub contributor scraper", "repository_url", contributorData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "contributor_count_last_error", contributorData.software() - .toString(), "software", null, null); + Utils.saveExceptionInDatabase( + "GitHub contributor scraper", + "repository_url", + contributorData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "contributor_count_last_error", + contributorData.software().toString(), + "id", + null, + null + ); } catch (RsdResponseException e) { - Utils.saveExceptionInDatabase("GitHub contributor scraper", "repository_url", contributorData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "contributor_count_last_error", contributorData.software() - .toString(), "software", scrapedAt, "contributor_count_scraped_at"); + Utils.saveExceptionInDatabase( + "GitHub contributor scraper", + "repository_url", + contributorData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "contributor_count_last_error", + contributorData.software().toString(), + "id", + scrapedAt, + "contributor_count_scraped_at" + ); } catch (Exception e) { - Utils.saveExceptionInDatabase("GitHub contributor scraper", "repository_url", contributorData.software(), e); - Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "contributor_count_last_error", contributorData.software() - .toString(), "software", scrapedAt, "contributor_count_scraped_at"); + Utils.saveExceptionInDatabase( + "GitHub contributor scraper", + "repository_url", + contributorData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "repository_url", + "contributor_count_last_error", + contributorData.software().toString(), + "id", + scrapedAt, + "contributor_count_scraped_at" + ); } }); futures[i] = future; @@ -80,8 +132,13 @@ private static void scrapeGitHub() { } private static void scrapeGitLab() { - PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITLAB); - Collection dataToScrape = softwareInfoRepository.contributorData(Config.maxRequestsGithub()); + PostgrestConnector softwareInfoRepository = new PostgrestConnector( + Config.backendBaseUrl() + "/repository_url", + CodePlatformProvider.GITLAB + ); + Collection dataToScrape = softwareInfoRepository.contributorData( + Config.maxRequestsGitLab() + ); CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; ZonedDateTime scrapedAt = ZonedDateTime.now(); int i = 0; @@ -95,21 +152,61 @@ private static void scrapeGitLab() { if (projectPath.endsWith("/")) projectPath = projectPath.substring(0, projectPath.length() - 1); Integer scrapedContributorData = new GitlabScraper(apiUrl, projectPath).contributorCount(); - ContributorDatabaseData updatedData = new ContributorDatabaseData(new BasicRepositoryData(contributorData.software(), null), scrapedContributorData, scrapedAt); + ContributorDatabaseData updatedData = new ContributorDatabaseData( + new BasicRepositoryData(contributorData.software(), null), + scrapedContributorData, + scrapedAt + ); softwareInfoRepository.saveContributorCount(updatedData); } catch (RsdRateLimitException e) { // in case we hit the rate limit, we don't update the scraped_at time, so it gets scraped first next time - Utils.saveExceptionInDatabase("GitLab contributor scraper", "repository_url", contributorData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "contributor_count_last_error", contributorData.software() - .toString(), "software", null, null); + Utils.saveExceptionInDatabase( + "GitLab contributor scraper", + "repository_url", + contributorData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "contributor_count_last_error", + contributorData.software().toString(), + "id", + null, + null + ); } catch (RsdResponseException e) { - Utils.saveExceptionInDatabase("GitLab contributor scraper", "repository_url", contributorData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "contributor_count_last_error", contributorData.software() - .toString(), "software", scrapedAt, "contributor_count_scraped_at"); + Utils.saveExceptionInDatabase( + "GitLab contributor scraper", + "repository_url", + contributorData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "contributor_count_last_error", + contributorData.software().toString(), + "id", + scrapedAt, + "contributor_count_scraped_at" + ); } catch (Exception e) { - Utils.saveExceptionInDatabase("GitLab contributor scraper", "repository_url", contributorData.software(), e); - Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "contributor_count_last_error", contributorData.software() - .toString(), "software", scrapedAt, "contributor_count_scraped_at"); + Utils.saveExceptionInDatabase( + "GitLab contributor scraper", + "repository_url", + contributorData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "repository_url", + "contributor_count_last_error", + contributorData.software().toString(), + "id", + scrapedAt, + "contributor_count_scraped_at" + ); } }); futures[i] = future; @@ -119,8 +216,11 @@ private static void scrapeGitLab() { } private static void scrape4tu() { - PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.FOURTU); - Collection dataToScrape = softwareInfoRepository.contributorData(Config.maxRequestsGithub()); + PostgrestConnector softwareInfoRepository = new PostgrestConnector( + Config.backendBaseUrl() + "/repository_url", + CodePlatformProvider.FOURTU + ); + Collection dataToScrape = softwareInfoRepository.contributorData(Config.maxRequests4tu()); CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; ZonedDateTime scrapedAt = ZonedDateTime.now(); int i = 0; @@ -132,21 +232,61 @@ private static void scrape4tu() { FourTuGitScraper fourTuGitScraper = new FourTuGitScraper(repoUrl); Integer scrapedContributorData = fourTuGitScraper.contributorCount(); - ContributorDatabaseData updatedData = new ContributorDatabaseData(new BasicRepositoryData(contributorData.software(), null), scrapedContributorData, scrapedAt); + ContributorDatabaseData updatedData = new ContributorDatabaseData( + new BasicRepositoryData(contributorData.software(), null), + scrapedContributorData, + scrapedAt + ); softwareInfoRepository.saveContributorCount(updatedData); } catch (RsdRateLimitException e) { // in case we hit the rate limit, we don't update the scraped_at time, so it gets scraped first next time - Utils.saveExceptionInDatabase("4TU contributor scraper", "repository_url", contributorData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "contributor_count_last_error", contributorData.software() - .toString(), "software", null, null); + Utils.saveExceptionInDatabase( + "4TU contributor scraper", + "repository_url", + contributorData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "contributor_count_last_error", + contributorData.software().toString(), + "id", + null, + null + ); } catch (RsdResponseException e) { - Utils.saveExceptionInDatabase("4TU contributor scraper", "repository_url", contributorData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "contributor_count_last_error", contributorData.software() - .toString(), "software", scrapedAt, "contributor_count_scraped_at"); + Utils.saveExceptionInDatabase( + "4TU contributor scraper", + "repository_url", + contributorData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "contributor_count_last_error", + contributorData.software().toString(), + "id", + scrapedAt, + "contributor_count_scraped_at" + ); } catch (Exception e) { - Utils.saveExceptionInDatabase("4TU contributor scraper", "repository_url", contributorData.software(), e); - Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "contributor_count_last_error", contributorData.software() - .toString(), "software", scrapedAt, "contributor_count_scraped_at"); + Utils.saveExceptionInDatabase( + "4TU contributor scraper", + "repository_url", + contributorData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "repository_url", + "contributor_count_last_error", + contributorData.software().toString(), + "id", + scrapedAt, + "contributor_count_scraped_at" + ); } }); futures[i] = future; diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainProgrammingLanguages.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainProgrammingLanguages.java index 7e7752d..c5e292e 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainProgrammingLanguages.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/MainProgrammingLanguages.java @@ -1,10 +1,16 @@ -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2025 Dusan Mijatovic (Netherlands eScience Center) // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.git; +import java.net.URI; +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; import nl.esciencecenter.rsd.scraper.Config; import nl.esciencecenter.rsd.scraper.RsdRateLimitException; import nl.esciencecenter.rsd.scraper.RsdResponseException; @@ -12,18 +18,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.URI; -import java.time.ZonedDateTime; -import java.util.Collection; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; - public class MainProgrammingLanguages { private static final Logger LOGGER = LoggerFactory.getLogger(MainProgrammingLanguages.class); public static void main(String[] args) { - LOGGER.info("Start scraping programming languages"); long t1 = System.currentTimeMillis(); @@ -31,6 +30,7 @@ public static void main(String[] args) { scrapeGithub(); scrapeGitLab(); scrape4tu(); + scrapeCodeberg(); long time = System.currentTimeMillis() - t1; @@ -38,7 +38,10 @@ public static void main(String[] args) { } private static void scrapeGitLab() { - PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITLAB); + PostgrestConnector softwareInfoRepository = new PostgrestConnector( + Config.backendBaseUrl() + "/repository_url", + CodePlatformProvider.GITLAB + ); Collection dataToScrape = softwareInfoRepository.languagesData(Config.maxRequestsGitLab()); CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; ZonedDateTime scrapedAt = ZonedDateTime.now(); @@ -53,20 +56,60 @@ private static void scrapeGitLab() { if (projectPath.endsWith("/")) projectPath = projectPath.substring(0, projectPath.length() - 1); String scrapedLanguages = new GitlabScraper(apiUrl, projectPath).languages(); - LanguagesData updatedData = new LanguagesData(new BasicRepositoryData(programmingLanguageData.software(), null), scrapedLanguages, scrapedAt); + LanguagesData updatedData = new LanguagesData( + new BasicRepositoryData(programmingLanguageData.software(), null), + scrapedLanguages, + scrapedAt + ); softwareInfoRepository.saveLanguagesData(updatedData); } catch (RsdRateLimitException e) { - Utils.saveExceptionInDatabase("GitLab programming languages scraper", "repository_url", programmingLanguageData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "languages_last_error", programmingLanguageData.software() - .toString(), "software", null, null); + Utils.saveExceptionInDatabase( + "GitLab programming languages scraper", + "repository_url", + programmingLanguageData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "languages_last_error", + programmingLanguageData.software().toString(), + "id", + null, + null + ); } catch (RsdResponseException e) { - Utils.saveExceptionInDatabase("GitLab programming languages scraper", "repository_url", programmingLanguageData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "languages_last_error", programmingLanguageData.software() - .toString(), "software", scrapedAt, "languages_scraped_at"); + Utils.saveExceptionInDatabase( + "GitLab programming languages scraper", + "repository_url", + programmingLanguageData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "languages_last_error", + programmingLanguageData.software().toString(), + "id", + scrapedAt, + "languages_scraped_at" + ); } catch (Exception e) { - Utils.saveExceptionInDatabase("GitLab programming languages scraper", "repository_url", programmingLanguageData.software(), e); - Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "languages_last_error", programmingLanguageData.software() - .toString(), "software", scrapedAt, "languages_scraped_at"); + Utils.saveExceptionInDatabase( + "GitLab programming languages scraper", + "repository_url", + programmingLanguageData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "repository_url", + "languages_last_error", + programmingLanguageData.software().toString(), + "id", + scrapedAt, + "languages_scraped_at" + ); } }); futures[i] = future; @@ -76,7 +119,10 @@ private static void scrapeGitLab() { } private static void scrapeGithub() { - PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.GITHUB); + PostgrestConnector softwareInfoRepository = new PostgrestConnector( + Config.backendBaseUrl() + "/repository_url", + CodePlatformProvider.GITHUB + ); Collection dataToScrape = softwareInfoRepository.languagesData(Config.maxRequestsGithub()); CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; ZonedDateTime scrapedAt = ZonedDateTime.now(); @@ -88,27 +134,74 @@ private static void scrapeGithub() { Optional githubScraperOptional = GithubScraper.create(repoUrl); if (githubScraperOptional.isEmpty()) { - Utils.saveErrorMessageInDatabase("Not a valid GitHub URL: " + repoUrl, "repository_url", "languages_last_error", programmingLanguageData.software() - .toString(), "software", scrapedAt, "languages_scraped_at"); + Utils.saveErrorMessageInDatabase( + "Not a valid GitHub URL: " + repoUrl, + "repository_url", + "languages_last_error", + programmingLanguageData.software().toString(), + "id", + scrapedAt, + "languages_scraped_at" + ); return; } GithubScraper githubScraper = githubScraperOptional.get(); String scrapedLanguages = githubScraper.languages(); - LanguagesData updatedData = new LanguagesData(new BasicRepositoryData(programmingLanguageData.software(), null), scrapedLanguages, scrapedAt); + LanguagesData updatedData = new LanguagesData( + new BasicRepositoryData(programmingLanguageData.software(), null), + scrapedLanguages, + scrapedAt + ); softwareInfoRepository.saveLanguagesData(updatedData); } catch (RsdRateLimitException e) { - Utils.saveExceptionInDatabase("GitHub programming languages scraper", "repository_url", programmingLanguageData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "languages_last_error", programmingLanguageData.software() - .toString(), "software", null, null); + Utils.saveExceptionInDatabase( + "GitHub programming languages scraper", + "repository_url", + programmingLanguageData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "languages_last_error", + programmingLanguageData.software().toString(), + "id", + null, + null + ); } catch (RsdResponseException e) { - Utils.saveExceptionInDatabase("GitHub programming languages scraper", "repository_url", programmingLanguageData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "languages_last_error", programmingLanguageData.software() - .toString(), "software", scrapedAt, "languages_scraped_at"); + Utils.saveExceptionInDatabase( + "GitHub programming languages scraper", + "repository_url", + programmingLanguageData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "languages_last_error", + programmingLanguageData.software().toString(), + "id", + scrapedAt, + "languages_scraped_at" + ); } catch (Exception e) { - Utils.saveExceptionInDatabase("GitHub programming languages scraper", "repository_url", programmingLanguageData.software(), e); - Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "languages_last_error", programmingLanguageData.software() - .toString(), "software", scrapedAt, "languages_scraped_at"); + Utils.saveExceptionInDatabase( + "GitHub programming languages scraper", + "repository_url", + programmingLanguageData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "repository_url", + "languages_last_error", + programmingLanguageData.software().toString(), + "id", + scrapedAt, + "languages_scraped_at" + ); } }); futures[i] = future; @@ -118,8 +211,11 @@ private static void scrapeGithub() { } private static void scrape4tu() { - PostgrestConnector softwareInfoRepository = new PostgrestConnector(Config.backendBaseUrl() + "/repository_url", CodePlatformProvider.FOURTU); - Collection dataToScrape = softwareInfoRepository.languagesData(10); + PostgrestConnector softwareInfoRepository = new PostgrestConnector( + Config.backendBaseUrl() + "/repository_url", + CodePlatformProvider.FOURTU + ); + Collection dataToScrape = softwareInfoRepository.languagesData(Config.maxRequests4tu()); CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; ZonedDateTime scrapedAt = ZonedDateTime.now(); int i = 0; @@ -130,20 +226,154 @@ private static void scrape4tu() { FourTuGitScraper fourTuGitScraper = new FourTuGitScraper(repoUrl); String scrapedLanguages = fourTuGitScraper.languages(); - LanguagesData updatedData = new LanguagesData(new BasicRepositoryData(programmingLanguageData.software(), null), scrapedLanguages, scrapedAt); + LanguagesData updatedData = new LanguagesData( + new BasicRepositoryData(programmingLanguageData.software(), null), + scrapedLanguages, + scrapedAt + ); + softwareInfoRepository.saveLanguagesData(updatedData); + } catch (RsdRateLimitException e) { + Utils.saveExceptionInDatabase( + "4TU programming languages scraper", + "repository_url", + programmingLanguageData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "languages_last_error", + programmingLanguageData.software().toString(), + "id", + null, + null + ); + } catch (RsdResponseException e) { + Utils.saveExceptionInDatabase( + "4TU programming languages scraper", + "repository_url", + programmingLanguageData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "languages_last_error", + programmingLanguageData.software().toString(), + "id", + scrapedAt, + "languages_scraped_at" + ); + } catch (Exception e) { + Utils.saveExceptionInDatabase( + "4TU programming languages scraper", + "repository_url", + programmingLanguageData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "repository_url", + "languages_last_error", + programmingLanguageData.software().toString(), + "id", + scrapedAt, + "languages_scraped_at" + ); + } + }); + futures[i] = future; + i++; + } + CompletableFuture.allOf(futures).join(); + } + + private static void scrapeCodeberg() { + PostgrestConnector softwareInfoRepository = new PostgrestConnector( + Config.backendBaseUrl() + "/repository_url", + CodePlatformProvider.CODEBERG + ); + Collection dataToScrape = softwareInfoRepository.languagesData( + Config.maxRequestsCodeberg() + ); + CompletableFuture[] futures = new CompletableFuture[dataToScrape.size()]; + ZonedDateTime scrapedAt = ZonedDateTime.now(); + int i = 0; + for (BasicRepositoryData programmingLanguageData : dataToScrape) { + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + String repoUrl = programmingLanguageData.url(); + + Optional codebergScraperOptional = CodebergScraper.fromUrl(repoUrl); + if (codebergScraperOptional.isEmpty()) { + Utils.saveErrorMessageInDatabase( + "Not a valid Codeberg URL: " + repoUrl, + "repository_url", + "languages_last_error", + programmingLanguageData.software().toString(), + "id", + scrapedAt, + "languages_scraped_at" + ); + return; + } + + CodebergScraper codebergScraper = codebergScraperOptional.get(); + String scrapedLanguages = codebergScraper.languages(); + LanguagesData updatedData = new LanguagesData( + new BasicRepositoryData(programmingLanguageData.software(), null), + scrapedLanguages, + scrapedAt + ); softwareInfoRepository.saveLanguagesData(updatedData); } catch (RsdRateLimitException e) { - Utils.saveExceptionInDatabase("4TU programming languages scraper", "repository_url", programmingLanguageData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "languages_last_error", programmingLanguageData.software() - .toString(), "software", null, null); + Utils.saveExceptionInDatabase( + "Codeberg programming languages scraper", + "repository_url", + programmingLanguageData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "languages_last_error", + programmingLanguageData.software().toString(), + "id", + null, + null + ); } catch (RsdResponseException e) { - Utils.saveExceptionInDatabase("4TU programming languages scraper", "repository_url", programmingLanguageData.software(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "repository_url", "languages_last_error", programmingLanguageData.software() - .toString(), "software", scrapedAt, "languages_scraped_at"); + Utils.saveExceptionInDatabase( + "Codeberg programming languages scraper", + "repository_url", + programmingLanguageData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "repository_url", + "languages_last_error", + programmingLanguageData.software().toString(), + "id", + scrapedAt, + "languages_scraped_at" + ); } catch (Exception e) { - Utils.saveExceptionInDatabase("4TU programming languages scraper", "repository_url", programmingLanguageData.software(), e); - Utils.saveErrorMessageInDatabase("Unknown error", "repository_url", "languages_last_error", programmingLanguageData.software() - .toString(), "software", scrapedAt, "languages_scraped_at"); + Utils.saveExceptionInDatabase( + "Codeberg programming languages scraper", + "repository_url", + programmingLanguageData.software(), + e + ); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "repository_url", + "languages_last_error", + programmingLanguageData.software().toString(), + "id", + scrapedAt, + "languages_scraped_at" + ); } }); futures[i] = future; diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnector.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnector.java index cdb626d..96e9bc3 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnector.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnector.java @@ -1,6 +1,7 @@ -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2026 Netherlands eScience Center // SPDX-FileCopyrightText: 2025 Christian Meeßen (GFZ) +// SPDX-FileCopyrightText: 2025 Dusan Mijatovic (Netherlands eScience Center) // SPDX-FileCopyrightText: 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences // // SPDX-License-Identifier: Apache-2.0 @@ -12,8 +13,6 @@ import com.google.gson.JsonNull; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.Utils; - import java.time.Instant; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -21,6 +20,7 @@ import java.util.Collection; import java.util.Objects; import java.util.UUID; +import nl.esciencecenter.rsd.scraper.Utils; public class PostgrestConnector { @@ -44,10 +44,24 @@ public PostgrestConnector(String backendUrl, CodePlatformProvider codePlatform) * @return The data corresponding to the git repositories of which the programming languages data were scraped the longest time ago */ public Collection languagesData(int limit) { - String data = Utils.getAsAdmin(backendUrl + "?" + filter + "&select=software,url&order=languages_scraped_at.asc.nullsfirst&limit=" + limit + "&" + Utils.atLeastOneHourAgoFilter("languages_scraped_at")); + String data = Utils.getAsAdmin( + backendUrl + + "?" + + filter + + "&select=id,url&order=languages_scraped_at.asc.nullsfirst&limit=" + + limit + + "&" + + Utils.atLeastOneHourAgoFilter("languages_scraped_at") + ); return parseBasicJsonData(data); } + public void saveCommitHistoryScrapedAt(UUID id, ZonedDateTime scrapedAt) { + JsonObject root = new JsonObject(); + root.addProperty("commit_history_scraped_at", scrapedAt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + Utils.patchAsAdmin(backendUrl + "?id=eq." + id, root.toString()); + } + /** * Fetch commit data from PostgREST * @@ -55,12 +69,28 @@ public Collection languagesData(int limit) { * @return The data corresponding to the git repositories of which the commit data were scraped the longest time ago */ public Collection commitData(int limit) { - String data = Utils.getAsAdmin(backendUrl + "?" + filter + "&select=software,url&order=commit_history_scraped_at.asc.nullsfirst&limit=" + limit + "&" + Utils.atLeastOneHourAgoFilter("commit_history_scraped_at")); + String data = Utils.getAsAdmin( + backendUrl + + "?" + + filter + + "&select=id,url&order=commit_history_scraped_at.asc.nullsfirst&limit=" + + limit + + "&" + + Utils.atLeastOneHourAgoFilter("commit_history_scraped_at") + ); return parseBasicJsonData(data); } public Collection commitDataWithHistory(int limit) { - String data = Utils.getAsAdmin(backendUrl + "?" + filter + "&select=software,url,commit_history_scraped_at,commit_history&order=commit_history_scraped_at.asc.nullsfirst&limit=" + limit + "&" + Utils.atLeastOneHourAgoFilter("commit_history_scraped_at")); + String data = Utils.getAsAdmin( + backendUrl + + "?" + + filter + + "&select=id,url,commit_history_scraped_at,commit_history&order=commit_history_scraped_at.asc.nullsfirst&limit=" + + limit + + "&" + + Utils.atLeastOneHourAgoFilter("commit_history_scraped_at") + ); return parseBasicJsonDataWithHistory(data); } @@ -71,12 +101,28 @@ public Collection commitDataWithHistory(int limi * @return The data corresponding to the git repositories of which the basic data were scraped the longest time ago */ public Collection statsData(int limit) { - String data = Utils.getAsAdmin(backendUrl + "?" + filter + "&select=software,url&order=basic_data_scraped_at.asc.nullsfirst&limit=" + limit + "&" + Utils.atLeastOneHourAgoFilter("basic_data_scraped_at")); + String data = Utils.getAsAdmin( + backendUrl + + "?" + + filter + + "&select=id,url&order=basic_data_scraped_at.asc.nullsfirst&limit=" + + limit + + "&" + + Utils.atLeastOneHourAgoFilter("basic_data_scraped_at") + ); return parseBasicJsonData(data); } public Collection contributorData(int limit) { - String data = Utils.getAsAdmin(backendUrl + "?" + filter + "&select=software,url&order=contributor_count_scraped_at.asc.nullsfirst&limit=" + limit + "&" + Utils.atLeastOneHourAgoFilter("contributor_count_scraped_at")); + String data = Utils.getAsAdmin( + backendUrl + + "?" + + filter + + "&select=id,url&order=contributor_count_scraped_at.asc.nullsfirst&limit=" + + limit + + "&" + + Utils.atLeastOneHourAgoFilter("contributor_count_scraped_at") + ); return parseBasicJsonData(data); } @@ -85,16 +131,22 @@ static Collection parseBasicJsonDataWithHistory( Collection result = new ArrayList<>(); for (JsonElement element : dataInArray) { JsonObject jsonObject = element.getAsJsonObject(); - String softwareUuid = jsonObject.getAsJsonPrimitive("software").getAsString(); + // TODO! replace software with id prop in BasicRepositoryData + // the software column is removed from repository_url table, use id or url instead + String softwareUuid = jsonObject.getAsJsonPrimitive("id").getAsString(); UUID software = UUID.fromString(softwareUuid); String url = jsonObject.getAsJsonPrimitive("url").getAsString(); String commitHistoryScrapedAtString = Utils.stringOrNull(jsonObject.get("commit_history_scraped_at")); - ZonedDateTime commitHistoryScrapedAt = commitHistoryScrapedAtString == null ? null : ZonedDateTime.parse(commitHistoryScrapedAtString); + ZonedDateTime commitHistoryScrapedAt = + commitHistoryScrapedAtString == null ? null : ZonedDateTime.parse(commitHistoryScrapedAtString); CommitsPerWeek commitsPerWeek = new CommitsPerWeek(); if (!jsonObject.get("commit_history").isJsonNull()) { JsonObject commitHistoryJsonObject = jsonObject.getAsJsonObject("commit_history"); for (String key : commitHistoryJsonObject.keySet()) { - commitsPerWeek.addCommits(Instant.ofEpochSecond(Long.parseLong(key)), commitHistoryJsonObject.getAsJsonPrimitive(key).getAsLong()); + commitsPerWeek.addCommits( + Instant.ofEpochSecond(Long.parseLong(key)), + commitHistoryJsonObject.getAsJsonPrimitive(key).getAsLong() + ); } } result.add(new BasicRepositoryDataWithHistory(software, url, commitHistoryScrapedAt, commitsPerWeek)); @@ -107,7 +159,9 @@ static Collection parseBasicJsonData(String data) { Collection result = new ArrayList<>(); for (JsonElement element : dataInArray) { JsonObject jsonObject = element.getAsJsonObject(); - String softwareUuid = jsonObject.getAsJsonPrimitive("software").getAsString(); + // TODO! replace software with id prop in BasicRepositoryData + // the software column is removed from repository_url table, use id or url instead + String softwareUuid = jsonObject.getAsJsonPrimitive("id").getAsString(); UUID software = UUID.fromString(softwareUuid); String url = jsonObject.getAsJsonPrimitive("url").getAsString(); @@ -117,51 +171,62 @@ static Collection parseBasicJsonData(String data) { } public void saveLanguagesData(LanguagesData languagesData) { - String json = String.format("{\"languages_last_error\": null, \"languages\": %s, \"languages_scraped_at\": \"%s\"}", languagesData.languages(), languagesData.languagesScrapedAt() - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - Utils.patchAsAdmin(backendUrl + "?software=eq." + languagesData.basicData().software().toString(), json); + String json = String.format( + "{\"languages_last_error\": null, \"languages\": %s, \"languages_scraped_at\": \"%s\"}", + languagesData.languages(), + languagesData.languagesScrapedAt().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + ); + Utils.patchAsAdmin(backendUrl + "?id=eq." + languagesData.basicData().software().toString(), json); } public void saveCommitData(CommitData commitData) { String json; if (commitData.commitHistory() == null) { - json = String.format("{\"commit_history_last_error\": null, \"commit_history_scraped_at\": \"%s\"}", commitData.commitHistoryScrapedAt() - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + json = String.format( + "{\"commit_history_last_error\": null, \"commit_history_scraped_at\": \"%s\"}", + commitData.commitHistoryScrapedAt().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + ); } else { commitData.commitHistory().addMissingZeros(); - json = String.format("{\"commit_history_last_error\": null, \"commit_history\": %s, \"commit_history_scraped_at\": \"%s\"}", commitData.commitHistory() - .toJson(), commitData.commitHistoryScrapedAt().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + json = String.format( + "{\"commit_history_last_error\": null, \"commit_history\": %s, \"commit_history_scraped_at\": \"%s\"}", + commitData.commitHistory().toJson(), + commitData.commitHistoryScrapedAt().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + ); } - Utils.patchAsAdmin(backendUrl + "?software=eq." + commitData.basicData().software().toString(), json); + Utils.patchAsAdmin(backendUrl + "?id=eq." + commitData.basicData().software().toString(), json); } public void saveBasicData(BasicGitDatabaseData basicData) { JsonObject jsonObject = new JsonObject(); jsonObject.add("basic_data_last_error", JsonNull.INSTANCE); - jsonObject.addProperty("basic_data_scraped_at", basicData.dataScrapedAt() - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + jsonObject.addProperty( + "basic_data_scraped_at", + basicData.dataScrapedAt().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + ); jsonObject.addProperty("archived", basicData.statsData().archived()); jsonObject.addProperty("license", basicData.statsData().license()); jsonObject.addProperty("star_count", basicData.statsData().starCount()); jsonObject.addProperty("fork_count", basicData.statsData().forkCount()); jsonObject.addProperty("open_issue_count", basicData.statsData().openIssueCount()); - Utils.patchAsAdmin(backendUrl + "?software=eq." + basicData.basicData() - .software() - .toString(), jsonObject.toString()); + Utils.patchAsAdmin(backendUrl + "?id=eq." + basicData.basicData().software().toString(), jsonObject.toString()); } public void saveContributorCount(ContributorDatabaseData contributorData) { JsonObject jsonObject = new JsonObject(); jsonObject.add("contributor_count_last_error", JsonNull.INSTANCE); - jsonObject.addProperty("contributor_count_scraped_at", contributorData.dataScrapedAt() - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + jsonObject.addProperty( + "contributor_count_scraped_at", + contributorData.dataScrapedAt().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + ); if (contributorData.contributorCount() != null) { jsonObject.addProperty("contributor_count", contributorData.contributorCount()); } - Utils.patchAsAdmin(backendUrl + "?software=eq." + contributorData.basicData() - .software() - .toString(), jsonObject.toString()); + Utils.patchAsAdmin( + backendUrl + "?id=eq." + contributorData.basicData().software().toString(), + jsonObject.toString() + ); } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepository.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepository.java index 96df10a..8325dad 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepository.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepository.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 @@ -10,29 +10,42 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.gson.reflect.TypeToken; -import nl.esciencecenter.rsd.scraper.RsdResponseException; -import nl.esciencecenter.rsd.scraper.Utils; - import java.io.IOException; import java.net.http.HttpResponse; import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; public class GitHubSpdxLicenseRepository { private static final Gson gson = new Gson(); - private GitHubSpdxLicenseRepository() { - } - - public static Map getLicensesByIdMap() throws IOException, InterruptedException, RsdResponseException { + private GitHubSpdxLicenseRepository() {} + + /** + * + * Download all SPDX licenses and return them as a map + * + * @return a map of SPDX licenses, where the key is the same as the {@link SpdxLicense#licenseId()} + * @throws IOException when the underlying HTTP call throws this exception + * @throws InterruptedException when the current thread is interrupted during the HTTP call + * @throws RsdResponseException when the HTTP response is not successful + */ + public static Map getLicensesByIdMap() + throws IOException, InterruptedException, RsdResponseException { String url = "https://raw.githubusercontent.com/spdx/license-list-data/refs/heads/main/json/licenses.json"; HttpResponse response = Utils.getAsHttpResponse(url); if (response.statusCode() != 200) { - throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response while getting SPDX licenses"); + throw new RsdResponseException( + response.statusCode(), + response.uri(), + response.body(), + "Unexpected response while getting SPDX licenses" + ); } String json = response.body(); @@ -42,12 +55,9 @@ public static Map getLicensesByIdMap() throws IOException, static Map parseLicensesJson(String json) { JsonObject root = JsonParser.parseString(json).getAsJsonObject(); JsonArray licensesJsonArray = root.getAsJsonArray("licenses"); - TypeToken> spdxListTypeToken = new TypeToken<>() { - }; + TypeToken> spdxListTypeToken = new TypeToken<>() {}; List licenses = gson.fromJson(licensesJsonArray, spdxListTypeToken); - return licenses - .stream() - .collect(Collectors.toMap(SpdxLicense::licenseId, Function.identity())); + return licenses.stream().collect(Collectors.toMap(SpdxLicense::licenseId, Function.identity())); } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/SpdxLicense.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/SpdxLicense.java index 7300426..54b2e2b 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/SpdxLicense.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/license/SpdxLicense.java @@ -5,9 +5,4 @@ package nl.esciencecenter.rsd.scraper.license; -public record SpdxLicense( - String licenseId, - String reference, - String name -) { -} +public record SpdxLicense(String licenseId, String reference, String name) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/MainNassa.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/MainNassa.java index 28378f2..55c21e3 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/MainNassa.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/MainNassa.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 @@ -8,16 +8,6 @@ import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.BibtexParser; -import nl.esciencecenter.rsd.scraper.Config; -import nl.esciencecenter.rsd.scraper.RsdResponseException; -import nl.esciencecenter.rsd.scraper.Utils; -import nl.esciencecenter.rsd.scraper.doi.ExternalMentionRecord; -import nl.esciencecenter.rsd.scraper.license.GitHubSpdxLicenseRepository; -import nl.esciencecenter.rsd.scraper.license.SpdxLicense; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.StringReader; import java.net.URI; import java.net.http.HttpClient; @@ -32,6 +22,15 @@ import java.util.Map; import java.util.Set; import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.BibtexParser; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; +import nl.esciencecenter.rsd.scraper.doi.ExternalMentionRecord; +import nl.esciencecenter.rsd.scraper.license.GitHubSpdxLicenseRepository; +import nl.esciencecenter.rsd.scraper.license.SpdxLicense; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class MainNassa { @@ -42,16 +41,13 @@ public class MainNassa { private static final String NASSA_SERVICE_NAME = "NASSA scraper"; // https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28 - public static void main(String[] args) { + public static void main() { if (!Config.isNassaScraperEnabled()) { return; } URI rootContentUri = URI.create("https://api.github.com/repos/Archaeology-ABM/NASSA-modules/contents/"); - HttpRequest httpRequest = HttpRequest - .newBuilder(rootContentUri) - .GET() - .build(); + HttpRequest httpRequest = HttpRequest.newBuilder(rootContentUri).GET().build(); HttpResponse httpResponse; try { @@ -61,9 +57,15 @@ public static void main(String[] args) { return; } if (httpResponse.statusCode() != 200) { - String message = "Got a non-200 response from the GitHub API querying then contents of the NASSA repository"; + String message = + "Got a non-200 response from the GitHub API querying then contents of the NASSA repository"; LOGGER.error(message); - RsdResponseException exceptionToSave = new RsdResponseException(httpResponse.statusCode(), rootContentUri, httpResponse.body(), message); + RsdResponseException exceptionToSave = new RsdResponseException( + httpResponse.statusCode(), + rootContentUri, + httpResponse.body(), + message + ); Utils.saveExceptionInDatabase(NASSA_SERVICE_NAME, null, null, exceptionToSave); return; } @@ -74,7 +76,7 @@ public static void main(String[] args) { String fileOrDirName = jsonElement.getAsJsonObject().getAsJsonPrimitive("name").getAsString(); String htmlUrl = jsonElement.getAsJsonObject().getAsJsonPrimitive("html_url").getAsString(); if (!DENY_LIST_MODULES.contains(fileOrDirName) && NASSA_MODULE_PATTERN.asPredicate().test(fileOrDirName)) { - modules.add(new String[]{fileOrDirName, htmlUrl}); + modules.add(new String[] { fileOrDirName, htmlUrl }); } } @@ -83,18 +85,26 @@ public static void main(String[] args) { for (String[] moduleInfo : modules) { try { // e.g. https://raw.githubusercontent.com/Archaeology-ABM/NASSA-modules/main/1870-Schliemann-001/NASSA.yml - URI ymlUri = URI.create("https://raw.githubusercontent.com/Archaeology-ABM/NASSA-modules/main/%s/NASSA.yml".formatted(moduleInfo[0])); - httpRequest = HttpRequest - .newBuilder(ymlUri) - .GET() - .build(); - - httpResponse = HTTP_CLIENT.send(httpRequest, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + URI ymlUri = URI.create( + "https://raw.githubusercontent.com/Archaeology-ABM/NASSA-modules/main/%s/NASSA.yml".formatted( + moduleInfo[0] + ) + ); + httpRequest = HttpRequest.newBuilder(ymlUri).GET().build(); + + httpResponse = HTTP_CLIENT.send( + httpRequest, + HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8) + ); StringReader ymlStringReader = new StringReader(httpResponse.body()); String longDescription = null; try { - URI readmeUri = URI.create("https://raw.githubusercontent.com/Archaeology-ABM/NASSA-modules/main/%s/README.md".formatted(moduleInfo[0])); + URI readmeUri = URI.create( + "https://raw.githubusercontent.com/Archaeology-ABM/NASSA-modules/main/%s/README.md".formatted( + moduleInfo[0] + ) + ); longDescription = NassaReadmeParser.downloadAndReturnDescriptionFromReadme(readmeUri, HTTP_CLIENT); } catch (Exception e) { Utils.saveExceptionInDatabase(NASSA_SERVICE_NAME, null, null, e); @@ -104,18 +114,23 @@ public static void main(String[] args) { } // e.g. https://raw.githubusercontent.com/Archaeology-ABM/NASSA-modules/main/1870-Schliemann-001/references.bib - URI bibUri = URI.create("https://raw.githubusercontent.com/Archaeology-ABM/NASSA-modules/main/%s/references.bib".formatted(moduleInfo[0])); - httpRequest = HttpRequest - .newBuilder(bibUri) - .GET() - .build(); - httpResponse = HTTP_CLIENT.send(httpRequest, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + URI bibUri = URI.create( + "https://raw.githubusercontent.com/Archaeology-ABM/NASSA-modules/main/%s/references.bib".formatted( + moduleInfo[0] + ) + ); + httpRequest = HttpRequest.newBuilder(bibUri).GET().build(); + httpResponse = HTTP_CLIENT.send( + httpRequest, + HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8) + ); StringReader bibStringReader = new StringReader(httpResponse.body()); Map mentions; try { mentions = BibtexParser.parse(bibStringReader); } catch (Exception e) { - String message = "Exception parsing the Bibtex file of the following module: " + Arrays.toString(moduleInfo); + String message = + "Exception parsing the Bibtex file of the following module: " + Arrays.toString(moduleInfo); Exception exceptionToSave = new Exception(message, e); Utils.saveExceptionInDatabase(NASSA_SERVICE_NAME, null, null, exceptionToSave); mentions = Map.of(); @@ -123,7 +138,11 @@ public static void main(String[] args) { mentionsOfModules.add(mentions); LOGGER.info("parsing module: {} at URL {}", moduleInfo[0], ymlUri); - NassaSoftwareEntry softwareEntry = NassaSoftwareEntry.fromYaml(ymlStringReader, moduleInfo[1], longDescription); + NassaSoftwareEntry softwareEntry = NassaSoftwareEntry.fromYaml( + ymlStringReader, + moduleInfo[1], + longDescription + ); modulesToSave.add(softwareEntry); LOGGER.info("done parsing module: {}", moduleInfo[0]); } catch (InterruptedException e) { @@ -154,7 +173,10 @@ public static void main(String[] args) { Map mentions = mentionsOfModules.get(i); LOGGER.info("saving module to database: {}", nassaSoftwareEntry.id); try { - Utils.postAsAdmin(Config.backendBaseUrl() + "/rpc/nassa_import", nassaSoftwareEntry.toRsdJson(mentions, licenseMap)); + Utils.postAsAdmin( + Config.backendBaseUrl() + "/rpc/software_import", + nassaSoftwareEntry.toRsdJson(mentions, licenseMap) + ); } catch (Exception e) { String message = "Exception handling the following module: " + nassaSoftwareEntry.id; Exception exceptionToSave = new Exception(message, e); diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaReadmeParser.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaReadmeParser.java index 9b8581b..bc65b54 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaReadmeParser.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaReadmeParser.java @@ -5,8 +5,6 @@ package nl.esciencecenter.rsd.scraper.nassa; -import nl.esciencecenter.rsd.scraper.RsdResponseException; - import java.io.IOException; import java.net.URI; import java.net.http.HttpClient; @@ -14,6 +12,7 @@ import java.net.http.HttpResponse; import java.util.Objects; import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.RsdResponseException; public class NassaReadmeParser { @@ -27,15 +26,25 @@ static String getLongDescriptionFromReadme(String content) throws RsdNassaParseE return split[1]; } - throw new RsdNassaParseException("Couldn't split description into two parts around expected header, resulting array has size %d instead of 2".formatted(split.length)); + throw new RsdNassaParseException( + "Couldn't split description into two parts around expected header, resulting array has size %d instead of 2".formatted( + split.length + ) + ); } - public static String downloadAndReturnDescriptionFromReadme(URI readmeUri, HttpClient client) throws IOException, InterruptedException, RsdResponseException, RsdNassaParseException { + public static String downloadAndReturnDescriptionFromReadme(URI readmeUri, HttpClient client) + throws IOException, InterruptedException, RsdResponseException, RsdNassaParseException { HttpRequest request = HttpRequest.newBuilder(readmeUri).build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); if (response.statusCode() >= 300) { - throw new RsdResponseException(response.statusCode(), readmeUri, response.body(), "Unexpected response getting NASSA README"); + throw new RsdResponseException( + response.statusCode(), + readmeUri, + response.body(), + "Unexpected response getting NASSA README" + ); } return getLongDescriptionFromReadme(response.body()); diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntry.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntry.java index ee12d39..2e3de62 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntry.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntry.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 @@ -8,13 +8,6 @@ import com.google.gson.JsonArray; import com.google.gson.JsonNull; import com.google.gson.JsonObject; -import nl.esciencecenter.rsd.scraper.doi.ExternalMentionRecord; -import nl.esciencecenter.rsd.scraper.doi.PostgrestMentionRepository; -import nl.esciencecenter.rsd.scraper.license.SpdxLicense; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.Yaml; - import java.io.Reader; import java.util.ArrayList; import java.util.Collection; @@ -22,6 +15,13 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import nl.esciencecenter.rsd.scraper.doi.ExternalMentionRecord; +import nl.esciencecenter.rsd.scraper.doi.PostgrestMentionRepository; +import nl.esciencecenter.rsd.scraper.git.CodePlatformProvider; +import nl.esciencecenter.rsd.scraper.license.SpdxLicense; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; public class NassaSoftwareEntry { @@ -159,23 +159,31 @@ public static NassaSoftwareEntry fromYaml(Reader yamlSource, String gitHtmlUrl, public String toRsdJson(Map mentions, Map licenseMap) { JsonObject root = new JsonObject(); + root.addProperty("community_slug", "nassa"); root.addProperty("slug", id.toLowerCase()); root.addProperty("brand_name", title); + root.add("concept_doi", JsonNull.INSTANCE); root.addProperty("description", descriptionWithInputsAndOutputs()); - root.addProperty("short_statement", shortDescription.length() > 300 ? shortDescription.substring(0, 297) + "..." : shortDescription); + root.add("description_url", JsonNull.INSTANCE); + root.addProperty( + "short_statement", + shortDescription.length() > 300 ? shortDescription.substring(0, 297) + "..." : shortDescription + ); root.addProperty("get_started_url", gitHtmlUrl + "/" + docsDir); root.addProperty("repository_url", gitHtmlUrl); + root.addProperty("code_platform", CodePlatformProvider.GITHUB.toDatabaseString()); + root.addProperty("scraping_disabled_reason", "This is a NASSA module which is not a repository root"); - if (license != null) { - SpdxLicense spdxLicense = licenseMap.get(license); - if (spdxLicense != null) { + { + SpdxLicense spdxLicense; + if (license != null && (spdxLicense = licenseMap.get(license)) != null) { root.addProperty("license_value", spdxLicense.licenseId()); root.addProperty("license_name", spdxLicense.name()); root.addProperty("license_url", spdxLicense.reference()); root.addProperty("license_open_source", true); } else { LOGGER.warn("Unexpected SPDX license: {}", license); - // we need to add this so we can call the nassa_import RPC with all parameters + // we need to add this so we can call the software_import RPC with all parameters root.add("license_value", JsonNull.INSTANCE); root.add("license_name", JsonNull.INSTANCE); root.add("license_url", JsonNull.INSTANCE); @@ -191,12 +199,14 @@ public String toRsdJson(Map mentions, Map mentions, Map mentions, Map implementation : implementations) { - if (implementation != null && implementation.get("language") != null && implementation.get("language") instanceof String lang) { + if ( + implementation != null && + implementation.get("language") != null && + implementation.get("language") instanceof String lang + ) { languagesJson.add(lang); } } @@ -273,6 +288,8 @@ public String toRsdJson(Map mentions, Map map, int position) { String emailAddress = (String) map.get("email"); String orcid = (String) map.get("orcid"); - return new NassaContributor( - givenName, - familyName, - emailAddress, - roles, - orcid, - position - ); + return new NassaContributor(givenName, familyName, emailAddress, roles, orcid, position); } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/BasicPackageManagerData.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/BasicPackageManagerData.java index 9e9f680..89cbc0f 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/BasicPackageManagerData.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/BasicPackageManagerData.java @@ -7,9 +7,4 @@ import java.util.UUID; -public record BasicPackageManagerData( - UUID id, - String url, - PackageManagerType type -) { -} +public record BasicPackageManagerData(UUID id, String url, PackageManagerType type) {} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/MainPackageManager.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/MainPackageManager.java index 8497d5a..cdcaa81 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/MainPackageManager.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/MainPackageManager.java @@ -7,6 +7,16 @@ package nl.esciencecenter.rsd.scraper.package_manager; +import java.io.IOException; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import nl.esciencecenter.rsd.scraper.Config; import nl.esciencecenter.rsd.scraper.RsdRateLimitException; import nl.esciencecenter.rsd.scraper.RsdResponseException; @@ -17,6 +27,8 @@ import nl.esciencecenter.rsd.scraper.package_manager.scrapers.DockerHubScraper; import nl.esciencecenter.rsd.scraper.package_manager.scrapers.FourTuScraper; import nl.esciencecenter.rsd.scraper.package_manager.scrapers.GoScraper; +import nl.esciencecenter.rsd.scraper.package_manager.scrapers.InvalidPackageManagerUrlException; +import nl.esciencecenter.rsd.scraper.package_manager.scrapers.JuliaScraper; import nl.esciencecenter.rsd.scraper.package_manager.scrapers.MavenScraper; import nl.esciencecenter.rsd.scraper.package_manager.scrapers.NpmScraper; import nl.esciencecenter.rsd.scraper.package_manager.scrapers.PackageManagerScraper; @@ -25,17 +37,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - public class MainPackageManager { private static final Logger LOGGER = LoggerFactory.getLogger(MainPackageManager.class); @@ -88,7 +89,8 @@ public static void main(String[] args) { LOGGER.info("Done scraping package manager data ({} ms.)", time); } - static PackageManagerScraper scraperForType(PackageManagerType type, String url) { + static PackageManagerScraper scraperForType(PackageManagerType type, String url) + throws InvalidPackageManagerUrlException { return switch (type) { case anaconda -> new AnacondaScraper(url); case cran -> new CranScraper(url); @@ -96,6 +98,7 @@ static PackageManagerScraper scraperForType(PackageManagerType type, String url) case dockerhub -> new DockerHubScraper(url); case fourtu -> new FourTuScraper(); case golang -> new GoScraper(url); + case julia -> new JuliaScraper(url); case maven, sonatype -> new MavenScraper(url); case npm -> new NpmScraper(url); case pypi -> new PypiScraper(url); @@ -104,7 +107,11 @@ static PackageManagerScraper scraperForType(PackageManagerType type, String url) }; } - static void scrapeDownloads(BasicPackageManagerData data, PostgrestConnector postgrestConnector, ZonedDateTime scrapedAt) { + static void scrapeDownloads( + BasicPackageManagerData data, + PostgrestConnector postgrestConnector, + ZonedDateTime scrapedAt + ) throws InterruptedException { String packageManagerUrl = data.url(); PackageManagerType type = data.type(); @@ -115,20 +122,45 @@ static void scrapeDownloads(BasicPackageManagerData data, PostgrestConnector pos postgrestConnector.saveDownloadCount(data.id(), downloads, scrapedAt); } catch (RsdRateLimitException e) { Utils.saveExceptionInDatabase("Package manager downloads scraper", "package_manager", data.id(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "package_manager", "download_count_last_error", data.id() - .toString(), "id", null, null); - } catch (RsdResponseException e) { + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "package_manager", + "download_count_last_error", + data.id().toString(), + "id", + null, + null + ); + } catch (RsdResponseException | InvalidPackageManagerUrlException e) { Utils.saveExceptionInDatabase("Package manager downloads scraper", "package_manager", data.id(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "package_manager", "download_count_last_error", data.id() - .toString(), "id", scrapedAt, "download_count_scraped_at"); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "package_manager", + "download_count_last_error", + data.id().toString(), + "id", + scrapedAt, + "download_count_scraped_at" + ); } catch (IOException e) { Utils.saveExceptionInDatabase("Package manager downloads scraper", "package_manager", data.id(), e); - Utils.saveErrorMessageInDatabase("Unknown error", "package_manager", "download_count_last_error", data.id() - .toString(), "id", scrapedAt, "download_count_scraped_at"); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "package_manager", + "download_count_last_error", + data.id().toString(), + "id", + scrapedAt, + "download_count_scraped_at" + ); } } - static void scrapeReverseDependencies(BasicPackageManagerData data, PostgrestConnector postgrestConnector, ZonedDateTime scrapedAt) { + static void scrapeReverseDependencies( + BasicPackageManagerData data, + PostgrestConnector postgrestConnector, + ZonedDateTime scrapedAt + ) throws InterruptedException { String packageManagerUrl = data.url(); PackageManagerType type = data.type(); @@ -138,17 +170,53 @@ static void scrapeReverseDependencies(BasicPackageManagerData data, PostgrestCon revDeps = scraper.reverseDependencies(); postgrestConnector.saveReverseDependencyCount(data.id(), revDeps, scrapedAt); } catch (RsdRateLimitException e) { - Utils.saveExceptionInDatabase("Package manager reverse dependencies scraper", "package_manager", data.id(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "package_manager", "reverse_dependency_count_last_error", data.id() - .toString(), "id", null, null); - } catch (RsdResponseException e) { - Utils.saveExceptionInDatabase("Package manager reverse dependencies scraper", "package_manager", data.id(), e); - Utils.saveErrorMessageInDatabase(e.getMessage(), "package_manager", "reverse_dependency_count_last_error", data.id() - .toString(), "id", scrapedAt, "reverse_dependency_count_scraped_at"); - } catch (Exception e) { - Utils.saveExceptionInDatabase("Package manager reverse dependencies scraper", "package_manager", data.id(), e); - Utils.saveErrorMessageInDatabase("Unknown error", "package_manager", "reverse_dependency_count_last_error", data.id() - .toString(), "id", scrapedAt, "reverse_dependency_count_scraped_at"); + Utils.saveExceptionInDatabase( + "Package manager reverse dependencies scraper", + "package_manager", + data.id(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "package_manager", + "reverse_dependency_count_last_error", + data.id().toString(), + "id", + null, + null + ); + } catch (RsdResponseException | InvalidPackageManagerUrlException e) { + Utils.saveExceptionInDatabase( + "Package manager reverse dependencies scraper", + "package_manager", + data.id(), + e + ); + Utils.saveErrorMessageInDatabase( + e.getMessage(), + "package_manager", + "reverse_dependency_count_last_error", + data.id().toString(), + "id", + scrapedAt, + "reverse_dependency_count_scraped_at" + ); + } catch (IOException e) { + Utils.saveExceptionInDatabase( + "Package manager reverse dependencies scraper", + "package_manager", + data.id(), + e + ); + Utils.saveErrorMessageInDatabase( + "Unknown error", + "package_manager", + "reverse_dependency_count_last_error", + data.id().toString(), + "id", + scrapedAt, + "reverse_dependency_count_scraped_at" + ); } } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PackageManagerType.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PackageManagerType.java index 91fbe03..d207c53 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PackageManagerType.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PackageManagerType.java @@ -14,10 +14,11 @@ public enum PackageManagerType { dockerhub, fourtu, golang, + julia, maven, npm, pixi, pypi, sonatype, - other + other, } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PostgrestConnector.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PostgrestConnector.java index 37273cd..8fe6e04 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PostgrestConnector.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/PostgrestConnector.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2023 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2026 Netherlands eScience Center // SPDX-FileCopyrightText: 2025 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences // SPDX-FileCopyrightText: 2025 Paula Stock (GFZ) // @@ -7,22 +7,38 @@ package nl.esciencecenter.rsd.scraper.package_manager; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Objects; +import java.util.StringJoiner; import java.util.UUID; - -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; - import nl.esciencecenter.rsd.scraper.Utils; public class PostgrestConnector { + private static final Collection downloadEnabledPackageManagers = List.of( + PackageManagerType.dockerhub, + PackageManagerType.julia, + PackageManagerType.pixi + ); + private static final Collection reverseDependencyEnabledPackageManagers = List.of( + PackageManagerType.anaconda, + PackageManagerType.cran, + PackageManagerType.crates, + PackageManagerType.golang, + PackageManagerType.maven, + PackageManagerType.npm, + PackageManagerType.pypi, + PackageManagerType.sonatype + ); + private final String backendUrl; public PostgrestConnector(String backendUrl) { @@ -30,26 +46,64 @@ public PostgrestConnector(String backendUrl) { } public Collection oldestDownloadCounts(int limit) { - String filter = "download_count_scraping_disabled_reason=is.null&or=(package_manager.eq.dockerhub,package_manager.eq.pixi)"; - String data = Utils.getAsAdmin(backendUrl + "?" + filter + "&select=id,url,package_manager&order=download_count_scraped_at.asc.nullsfirst&limit=" + limit - + "&" + Utils.atLeastOneHourAgoFilter("download_count_scraped_at") + String packageManagerFilter = PostgrestConnector.joinPackageManagersAsFilter(downloadEnabledPackageManagers); + + String filter = "download_count_scraping_disabled_reason=is.null&" + packageManagerFilter; + String data = Utils.getAsAdmin( + backendUrl + + "?" + + filter + + "&select=id,url,package_manager&order=download_count_scraped_at.asc.nullsfirst&limit=" + + limit + + "&" + + Utils.atLeastOneHourAgoFilter("download_count_scraped_at") ); return parseBasicJsonData(data); } public Collection oldestReverseDependencyCounts(int limit) { - String filter = "reverse_dependency_count_scraping_disabled_reason=is.null&or=(package_manager.eq.anaconda,package_manager.eq.cran,package_manager.eq.crates,package_manager.eq.golang,package_manager.eq.maven,package_manager.eq.npm,package_manager.eq.pypi,package_manager.eq.sonatype)"; - String data = Utils.getAsAdmin(backendUrl + "?" + filter + "&select=id,url,package_manager&order=reverse_dependency_count_scraped_at.asc.nullsfirst&limit=" + limit + "&" + Utils.atLeastOneHourAgoFilter("reverse_dependency_count_scraped_at")); + String packageManagerFilter = PostgrestConnector.joinPackageManagersAsFilter( + reverseDependencyEnabledPackageManagers + ); + + String filter = "reverse_dependency_count_scraping_disabled_reason=is.null&" + packageManagerFilter; + String data = Utils.getAsAdmin( + backendUrl + + "?" + + filter + + "&select=id,url,package_manager&order=reverse_dependency_count_scraped_at.asc.nullsfirst&limit=" + + limit + + "&" + + Utils.atLeastOneHourAgoFilter("reverse_dependency_count_scraped_at") + ); return parseBasicJsonData(data); } + private static String joinPackageManagersAsFilter(Collection packageManagerTypes) { + StringJoiner packageManagerJoiner = new StringJoiner(",", "or=(", ")"); + + for (PackageManagerType packageManagerType : packageManagerTypes) { + packageManagerJoiner.add("package_manager.eq." + packageManagerType.toString()); + } + + return packageManagerJoiner.toString(); + } + public void saveDownloadCount(UUID id, Long count, ZonedDateTime scrapedAt) { - String json = "{\"download_count\": %s, \"download_count_scraped_at\": \"%s\", \"download_count_last_error\": null}".formatted(count, scrapedAt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + String json = + "{\"download_count\": %s, \"download_count_scraped_at\": \"%s\", \"download_count_last_error\": null}".formatted( + count, + scrapedAt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + ); Utils.patchAsAdmin(backendUrl + "?id=eq." + id, json); } public void saveReverseDependencyCount(UUID id, Integer count, ZonedDateTime scrapedAt) { - String json = "{\"reverse_dependency_count\": %s, \"reverse_dependency_count_scraped_at\": \"%s\", \"reverse_dependency_count_last_error\": null}".formatted(count, scrapedAt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + String json = + "{\"reverse_dependency_count\": %s, \"reverse_dependency_count_scraped_at\": \"%s\", \"reverse_dependency_count_last_error\": null}".formatted( + count, + scrapedAt.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + ); Utils.patchAsAdmin(backendUrl + "?id=eq." + id, json); } @@ -61,7 +115,9 @@ Collection parseBasicJsonData(String json) { String idString = jsonObject.getAsJsonPrimitive("id").getAsString(); UUID id = UUID.fromString(idString); String url = jsonObject.getAsJsonPrimitive("url").getAsString(); - PackageManagerType type = PackageManagerType.valueOf(jsonObject.getAsJsonPrimitive("package_manager").getAsString()); + PackageManagerType type = PackageManagerType.valueOf( + jsonObject.getAsJsonPrimitive("package_manager").getAsString() + ); result.add(new BasicPackageManagerData(id, url, type)); } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/AnacondaScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/AnacondaScraper.java index 581763d..c85e86e 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/AnacondaScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/AnacondaScraper.java @@ -7,13 +7,11 @@ import com.google.gson.JsonElement; import com.google.gson.JsonParser; - -import nl.esciencecenter.rsd.scraper.RsdResponseException; - import java.io.IOException; import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.RsdResponseException; public class AnacondaScraper implements PackageManagerScraper { diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraper.java index 826f4f9..32cba96 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraper.java @@ -7,19 +7,23 @@ import com.google.gson.JsonElement; import com.google.gson.JsonParser; - -import nl.esciencecenter.rsd.scraper.RsdResponseException; - import java.io.IOException; import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.RsdResponseException; public class CranScraper implements PackageManagerScraper { final String packageName; - private static final Pattern urlPattern1 = Pattern.compile("https://cran\\.r-project\\.org/web/packages/([^/\\s]+)/?(index\\.html/?)?", Pattern.CASE_INSENSITIVE); - private static final Pattern urlPattern2 = Pattern.compile("https://cran\\.r-project\\.org/package=([^/\\s]+)/?", Pattern.CASE_INSENSITIVE); + private static final Pattern urlPattern1 = Pattern.compile( + "https://cran\\.r-project\\.org/web/packages/([^/\\s]+)/?(index\\.html/?)?", + Pattern.CASE_INSENSITIVE + ); + private static final Pattern urlPattern2 = Pattern.compile( + "https://cran\\.r-project\\.org/package=([^/\\s]+)/?", + Pattern.CASE_INSENSITIVE + ); public CranScraper(String url) { Objects.requireNonNull(url); diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraper.java index ab3db87..6384d58 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraper.java @@ -7,12 +7,11 @@ import com.google.gson.JsonElement; import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.RsdResponseException; - import java.io.IOException; import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.RsdResponseException; public class CratesScraper implements PackageManagerScraper { diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/DockerHubScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/DockerHubScraper.java index f5a034c..68f3071 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/DockerHubScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/DockerHubScraper.java @@ -6,8 +6,6 @@ package nl.esciencecenter.rsd.scraper.package_manager.scrapers; import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.RsdResponseException; - import java.io.IOException; import java.net.URI; import java.net.http.HttpClient; @@ -17,6 +15,7 @@ import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.RsdResponseException; public class DockerHubScraper implements PackageManagerScraper { @@ -40,19 +39,19 @@ public Long downloads() throws RsdResponseException { String url; if (owner.equals("_")) url = "https://hub.docker.com/v2/repositories/library/" + packageName; else url = "https://hub.docker.com/v2/repositories/" + owner + "/" + packageName; - HttpClient client = HttpClient.newBuilder() - .followRedirects(HttpClient.Redirect.NORMAL) - .build(); - HttpRequest request = HttpRequest.newBuilder(URI.create(url)) - .timeout(Duration.ofSeconds(30)) - .build(); + HttpClient client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build(); + HttpRequest request = HttpRequest.newBuilder(URI.create(url)).timeout(Duration.ofSeconds(30)).build(); String json; try { HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); json = switch (response.statusCode()) { case 200 -> response.body(); - default -> - throw new RsdResponseException(response.statusCode(), request.uri(), response.body(), "Unexpected response"); + default -> throw new RsdResponseException( + response.statusCode(), + request.uri(), + response.body(), + "Unexpected response" + ); }; } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/FourTuScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/FourTuScraper.java index 5a42818..e88ff7c 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/FourTuScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/FourTuScraper.java @@ -5,11 +5,11 @@ package nl.esciencecenter.rsd.scraper.package_manager.scrapers; -import nl.esciencecenter.rsd.scraper.RsdResponseException; - import java.io.IOException; +import nl.esciencecenter.rsd.scraper.RsdResponseException; public class FourTuScraper implements PackageManagerScraper { + @Override public Long downloads() throws IOException, RsdResponseException { throw new UnsupportedOperationException(); diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraper.java index 18f4c1a..270e9a8 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraper.java @@ -7,13 +7,12 @@ import com.google.gson.JsonElement; import com.google.gson.JsonParser; -import nl.esciencecenter.rsd.scraper.RsdResponseException; -import nl.esciencecenter.rsd.scraper.Utils; - import java.io.IOException; import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; public class GoScraper implements PackageManagerScraper { @@ -41,7 +40,9 @@ public Long downloads() { // Example URL: https://libraries.io/api/go/google.golang.org%2Fgrpc @Override public Integer reverseDependencies() throws IOException, InterruptedException, RsdResponseException { - String data = PackageManagerScraper.doLibrariesIoRequest("https://libraries.io/api/go/" + Utils.urlEncode(packageName)); + String data = PackageManagerScraper.doLibrariesIoRequest( + "https://libraries.io/api/go/" + Utils.urlEncode(packageName) + ); JsonElement tree = JsonParser.parseString(data); return tree.getAsJsonObject().getAsJsonPrimitive("dependents_count").getAsInt(); } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/InvalidPackageManagerUrlException.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/InvalidPackageManagerUrlException.java new file mode 100644 index 0000000..3b86ced --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/InvalidPackageManagerUrlException.java @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +public class InvalidPackageManagerUrlException extends Exception { + + public InvalidPackageManagerUrlException(String message) { + super(message); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/JuliaScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/JuliaScraper.java new file mode 100644 index 0000000..93ed31a --- /dev/null +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/JuliaScraper.java @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.package_manager.scrapers; + +import com.google.gson.JsonParser; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.RsdResponseException; + +public class JuliaScraper implements PackageManagerScraper { + + private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(10); + private final String packageName; + private static final Pattern generalRegistryPattern = Pattern.compile( + "^https://github\\.com/JuliaRegistries/General/tree/master/[A-Z]/(\\w+)$" + ); + private static final Pattern ownRepoPattern = Pattern.compile("^https://github\\.com/[^/]+/(\\w+)\\.jl\\.git$"); + + public JuliaScraper(String url) throws InvalidPackageManagerUrlException { + Objects.requireNonNull(url); + + Matcher generalRegistryMatcher = generalRegistryPattern.matcher(url); + if (generalRegistryMatcher.find()) { + this.packageName = generalRegistryMatcher.group(1); + return; + } + + Matcher ownRepoMatcher = ownRepoPattern.matcher(url); + if (ownRepoMatcher.find()) { + this.packageName = ownRepoMatcher.group(1); + return; + } + + throw new InvalidPackageManagerUrlException(url); + } + + @Override + public Long downloads() throws IOException, RsdResponseException, InterruptedException { + // example: https://juliapkgstats.com/api/v1/total_downloads/Makie + URI url = URI.create("https://juliapkgstats.com/api/v1/total_downloads/" + packageName); + + try (HttpClient client = HttpClient.newBuilder().connectTimeout(DEFAULT_TIMEOUT).build()) { + HttpRequest request = HttpRequest.newBuilder(url).timeout(DEFAULT_TIMEOUT).build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + throw new RsdResponseException( + response.statusCode(), + response.uri(), + response.body(), + "Unexpected response getting Julia downloads for package: " + packageName + ); + } + + String numberAsString = JsonParser.parseString(response.body()) + .getAsJsonObject() + .getAsJsonPrimitive("total_requests") + .getAsString() + .replace(",", ""); + + return Long.parseLong(numberAsString); + } + } + + @Override + public Integer reverseDependencies() throws IOException, InterruptedException, RsdResponseException { + // Although libraries.io supports Julia, it either returns 0 or doesn't have the package + throw new UnsupportedOperationException(); + } +} diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraper.java index bbd74a5..6008cdc 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraper.java @@ -7,20 +7,20 @@ import com.google.gson.JsonElement; import com.google.gson.JsonParser; - -import nl.esciencecenter.rsd.scraper.RsdResponseException; - import java.io.IOException; import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.RsdResponseException; public class MavenScraper implements PackageManagerScraper { final String groupId; final String artifactId; private static final Pattern mvnPattern = Pattern.compile("https://mvnrepository\\.com/artifact/([^/]+)/([^/]+)/?"); - private static final Pattern sonatypePattern = Pattern.compile("https://central\\.sonatype\\.com/artifact/([^/]+)/([^/]+)/?"); + private static final Pattern sonatypePattern = Pattern.compile( + "https://central\\.sonatype\\.com/artifact/([^/]+)/([^/]+)/?" + ); public MavenScraper(String url) { Objects.requireNonNull(url); @@ -50,7 +50,9 @@ public Long downloads() { // Example URL: https://libraries.io/api/maven/io.github.sanctuuary:APE @Override public Integer reverseDependencies() throws IOException, InterruptedException, RsdResponseException { - String data = PackageManagerScraper.doLibrariesIoRequest("https://libraries.io/api/maven/" + groupId + ":" + artifactId); + String data = PackageManagerScraper.doLibrariesIoRequest( + "https://libraries.io/api/maven/" + groupId + ":" + artifactId + ); JsonElement tree = JsonParser.parseString(data); return tree.getAsJsonObject().getAsJsonPrimitive("dependents_count").getAsInt(); } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/NpmScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/NpmScraper.java index de89f2b..c7defe7 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/NpmScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/NpmScraper.java @@ -7,14 +7,12 @@ import com.google.gson.JsonElement; import com.google.gson.JsonParser; - -import nl.esciencecenter.rsd.scraper.RsdResponseException; -import nl.esciencecenter.rsd.scraper.Utils; - import java.io.IOException; import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.RsdResponseException; +import nl.esciencecenter.rsd.scraper.Utils; public class NpmScraper implements PackageManagerScraper { @@ -38,7 +36,9 @@ public Long downloads() { @Override public Integer reverseDependencies() throws IOException, InterruptedException, RsdResponseException { - String data = PackageManagerScraper.doLibrariesIoRequest("https://libraries.io/api/npm/" + Utils.urlEncode(packageName)); + String data = PackageManagerScraper.doLibrariesIoRequest( + "https://libraries.io/api/npm/" + Utils.urlEncode(packageName) + ); JsonElement tree = JsonParser.parseString(data); return tree.getAsJsonObject().getAsJsonPrimitive("dependents_count").getAsInt(); } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PackageManagerScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PackageManagerScraper.java index 6d8481b..6046fa8 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PackageManagerScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PackageManagerScraper.java @@ -1,14 +1,10 @@ -// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2023 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2025 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.package_manager.scrapers; -import nl.esciencecenter.rsd.scraper.Config; -import nl.esciencecenter.rsd.scraper.RsdRateLimitException; -import nl.esciencecenter.rsd.scraper.RsdResponseException; - import java.io.IOException; import java.net.URI; import java.net.http.HttpClient; @@ -16,10 +12,12 @@ import java.net.http.HttpResponse; import java.time.Duration; import java.util.Optional; +import nl.esciencecenter.rsd.scraper.Config; +import nl.esciencecenter.rsd.scraper.RsdRateLimitException; +import nl.esciencecenter.rsd.scraper.RsdResponseException; public interface PackageManagerScraper { - - Long downloads() throws IOException, RsdResponseException; + Long downloads() throws IOException, InterruptedException, RsdResponseException; Integer reverseDependencies() throws IOException, InterruptedException, RsdResponseException; @@ -27,22 +25,29 @@ static String doLibrariesIoRequest(String url) throws IOException, InterruptedEx Optional optionalApiKey = Config.librariesIoKey(); if (optionalApiKey.isPresent()) url += "?api_key=" + optionalApiKey.get(); - - HttpRequest request = HttpRequest.newBuilder(URI.create(url)) - .timeout(Duration.ofSeconds(30)) - .build(); - try (HttpClient client = HttpClient.newBuilder() - .followRedirects(HttpClient.Redirect.NORMAL) - .build();) { + HttpRequest request = HttpRequest.newBuilder(URI.create(url)).timeout(Duration.ofSeconds(30)).build(); + try (HttpClient client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build()) { HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return switch (response.statusCode()) { - case 429 -> - throw new RsdRateLimitException(429, request.uri(), response.body(), "Rate limit reached for libraries.io"); - case 404 -> - throw new RsdResponseException(404, request.uri(), response.body(), "Not found, is the URL correct?"); + case 429 -> throw new RsdRateLimitException( + 429, + request.uri(), + response.body(), + "Rate limit reached for libraries.io" + ); + case 404 -> throw new RsdResponseException( + 404, + request.uri(), + response.body(), + "Not found, is the URL correct?" + ); case 200 -> response.body(); - default -> - throw new RsdResponseException(response.statusCode(), response.uri(), response.body(), "Unexpected response"); + default -> throw new RsdResponseException( + response.statusCode(), + response.uri(), + response.body(), + "Unexpected response" + ); }; } } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PixiScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PixiScraper.java index 6c8ff1d..73449cc 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PixiScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PixiScraper.java @@ -5,15 +5,13 @@ package nl.esciencecenter.rsd.scraper.package_manager.scrapers; -import java.util.Objects; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; - +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import nl.esciencecenter.rsd.scraper.RsdResponseException; import nl.esciencecenter.rsd.scraper.Utils; @@ -36,23 +34,33 @@ public PixiScraper(String url) { @Override public Long downloads() throws RsdResponseException { - String graphqlQuery = String.format(""" - { - package(channelName: "%s", name: "%s") { - downloadCounts { - version - count - } + String graphqlQuery = String.format( + """ + { + package(channelName: "%s", name: "%s") { + downloadCounts { + version + count } } - """, - channelName, - packageName); + } + """, + channelName, + packageName + ); JsonObject body = new JsonObject(); body.addProperty("query", graphqlQuery); - String response = Utils.post("https://prefix.dev/api/graphql", body.toString(), "Content-Type", "application/json"); + String response = Utils.post( + "https://prefix.dev/api/graphql", + body.toString(), + "Content-Type", + "application/json" + ); JsonObject jsonObject = JsonParser.parseString(response).getAsJsonObject(); - JsonArray countItems = jsonObject.getAsJsonObject("data").getAsJsonObject("package").getAsJsonArray("downloadCounts"); + JsonArray countItems = jsonObject + .getAsJsonObject("data") + .getAsJsonObject("package") + .getAsJsonArray("downloadCounts"); long downloadsSum = 0; for (JsonElement item : countItems) { @@ -67,5 +75,4 @@ public Long downloads() throws RsdResponseException { public Integer reverseDependencies() { throw new UnsupportedOperationException(); } - } diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PypiScraper.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PypiScraper.java index 9bd0765..ba8153b 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PypiScraper.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/PypiScraper.java @@ -7,13 +7,11 @@ import com.google.gson.JsonElement; import com.google.gson.JsonParser; - -import nl.esciencecenter.rsd.scraper.RsdResponseException; - import java.io.IOException; import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; +import nl.esciencecenter.rsd.scraper.RsdResponseException; public class PypiScraper implements PackageManagerScraper { diff --git a/scrapers/src/main/resources/oaipmh.py b/scrapers/src/main/resources/oaipmh.py deleted file mode 100644 index 6179bae..0000000 --- a/scrapers/src/main/resources/oaipmh.py +++ /dev/null @@ -1,142 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) -# SPDX-FileCopyrightText: 2022 Netherlands eScience Center -# -# SPDX-License-Identifier: Apache-2.0 - -import json -import xml.etree.ElementTree as Tree -import os -from datetime import datetime, timedelta -import requests -from requests.exceptions import HTTPError -import time -import jwt - - -# Still need to enrich and correct the metadata. Currently serving Zenodo's metadata as-is. -def list_records(dois): - oaipmh_elem = _build_oaipmh_elem() - - # get the conceptdois from /api/software - backend_url = os.environ.get('POSTGREST_URL') - response = requests.get(backend_url + '/software?select=concept_doi') - - if dois is None: - softwares = response.json() - else: - softwares = [software for software in response.json() if software["concept_doi"] in dois] - - headers = { - 'Authorization': 'Bearer ' + os.environ.get('ZENODO_ACCESS_TOKEN') - } - - n_softwares = len(softwares) - - for i_software, software in enumerate(softwares): - - if software["concept_doi"] is None: - print(" %d/%d: concept_doi is None" % (i_software + 1, n_softwares)) - continue - - try: - redirect_url = _get_redirect(software) - identifier = _get_zenodo_identifier(redirect_url, headers) - url = 'https://zenodo.org/oai2d?verb=GetRecord&identifier=oai:zenodo.org:' + identifier + \ - '&metadataPrefix=datacite4' - record_elem = _get_datacite(url, headers) - oaipmh_elem.find('{http://www.openarchives.org/OAI/2.0/}ListRecords') \ - .append(record_elem) - - print( - " %d/%d: retrieved datacite4 metadata for %s" % (i_software + 1, n_softwares, software["concept_doi"]) - ) - - except requests.exceptions.RequestException as e: - print( - " %d/%d: There was an error while retrieving OAI-PMH metadata for https://doi.org/%s. %s" % - (i_software + 1, n_softwares, software["concept_doi"], str(e)) - ) - continue - - document = Tree.ElementTree(oaipmh_elem) - xml_str = Tree.tostring(document.getroot(), encoding='UTF-8', method='xml', xml_declaration=True).decode("utf8") - data_dict = {"data": xml_str} - - jwt_secret = os.environ.get('PGRST_JWT_SECRET') - jwt_token = jwt.encode({"role": "rsd_admin", "exp": datetime.now() + timedelta(minutes = 1)}, jwt_secret, algorithm="HS256") - patch_response = requests.patch('{}/oaipmh?id=is.true'.format(backend_url), json.dumps(data_dict), headers={'Authorization': 'Bearer ' + jwt_token}) - if patch_response.status_code != 204: - print(patch_response.text) - patch_response.raise_for_status() - - -def _build_oaipmh_elem(): - root_elem = Tree.Element('{http://www.openarchives.org/OAI/2.0/}OAI-PMH') - - response_date_elem = Tree.Element('{http://www.openarchives.org/OAI/2.0/}responseDate') - response_date_elem.text = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z' - root_elem.append(response_date_elem) - - request_elem = Tree.Element('{http://www.openarchives.org/OAI/2.0/}request') - request_elem.set('verb', 'ListRecords') - request_elem.set('metadataPrefix', 'datacite4') - request_elem.text = 'https://zenodo.org/oai2d' - root_elem.append(request_elem) - - listrecords_elem = Tree.Element('{http://www.openarchives.org/OAI/2.0/}ListRecords') - root_elem.append(listrecords_elem) - return root_elem - - -def _get_redirect(software): - response = requests.head('https://doi.org/{conceptdoi}'.format(conceptdoi=software["concept_doi"])) - if response.status_code == 302: - return response.next.url - elif response.status_code == 429: - print("HttpError 429 while trying to redirect from doi.org.") - response.raise_for_status() - else: - raise HTTPError( - "Expected a redirect from doi.org to zenodo.org, got {0} instead." - .format(response.status_code) - ) - - -def _get_zenodo_identifier(redirect_url, headers): - response = requests.head(redirect_url, headers=headers) - - if response.status_code == 302: - return response.next.url.split('/')[-1:][0] - elif response.status_code == 429: - print("HttpError 429 while trying to retrieve a record from zenodo.org.") - epoch_rate_limit_reset = int(response.headers.get("X-RateLimit-Reset")) - epoch_now = time.time() - sleep_time = max(0, epoch_rate_limit_reset - epoch_now) - print("Sleeping until: {} ({} seconds)".format(epoch_rate_limit_reset, sleep_time)) - time.sleep(sleep_time) - return _get_zenodo_identifier(redirect_url, headers) - else: - raise HTTPError( - "Expected a redirect from a conceptdoi to a versioned doi, got {0} instead." - .format(response.status_code) - ) - - -def _get_datacite(url, headers): - response = requests.get(url, headers=headers) - - if response.status_code != requests.codes.ok: - print("Error while trying to retrieve datacite metadata from zenodo.org.") - response.raise_for_status() - - # future work: correct/expand the datacite4 representation with other data from the RSDjson - return Tree.fromstring(response.text) \ - .find("{http://www.openarchives.org/OAI/2.0/}GetRecord") \ - .find("{http://www.openarchives.org/OAI/2.0/}record") - - -if __name__ == "__main__": - if bool(os.environ.get('ENABLE_OAIPMH_SCRAPER')): - list_records(None) - else: - print("The oaipmh scraper is disabled, aborting") diff --git a/scrapers/src/main/resources/releases.py b/scrapers/src/main/resources/releases.py deleted file mode 100644 index 0169199..0000000 --- a/scrapers/src/main/resources/releases.py +++ /dev/null @@ -1,310 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) -# SPDX-FileCopyrightText: 2022 Netherlands eScience Center -# -# SPDX-License-Identifier: Apache-2.0 - -from datetime import datetime -import re -import os -import time - -import requests -from cffconvert import Citation -import jwt -from datetime import datetime, timedelta -import json - - -class ReleaseScraper: - - """ - 1. check that the doi is a zenodo doi - 2. check that the doi is a concept doi - 3. retrieve versioned dois - foreach versioned doi - 4. visit the github release - 5. on github, check if there is a cff - foreach cff - 6. validate the cff - 7. generate bibtex, ris, endnote, and schema.org strings - - After initialization, the ReleaseScraper instance has a field .releases which is - an array of objects with one of the following layouts: - { - "citability": "full" - "datePublished": "2018-03-16" - "doi": "10.5281/zenodo.1200251", - "files": { - "bibtex": "file contents", - "cff": "file contents", - "schema_dot_org": "file contents", - "endnote": "file contents", - "ris": "file contents" - }, - "tag": "2.6.1" - } - or in case there is no valid CITATION.cff for a given release: - { - "citability": "doi-only" - "datePublished": "2018-02-23" - "doi": "10.5281/zenodo.1183426", - "files": {}, - "tag": "2.4.0" - } - """ - - def __init__(self, doi): - self.latest_schema_dot_org = None - self.releases = list() - self.is_citable = False - self.zenodo_data = dict(conceptdoi=None, versioned_dois=None) - self.title = None - self.message = None - self.doi = None - - if doi is None: - self.message = "record has no doi value of any kind." - return - - if not isinstance(doi, str): - self.message = "doi should be a string." - return - - if doi == "": - self.message = "doi should not be an empty string." - return - - url = "https://doi.org/{0}".format(doi) - response = requests.get(url) - if response.status_code == requests.codes.ok: - self.is_citable = True - self.doi = doi - else: - self.message = "error {0} resolving doi.".format(response.status_code) - return - - if not self.is_zenodo_doi(): - self.message = "doi is not a Zenodo doi." - return - - self.fetch_zenodo_data_conceptdoi() - if not self.is_concept_doi(): - self.message = "doi is not a concept doi." - return - - self.fetch_zenodo_data_versioned_dois() - if "hits" not in self.zenodo_data["versioned_dois"]: - self.message = "record doesn't seem to be resulting from GitHub-Zenodo integration." - return - - self.filter_zenodo_data_versioned_dois() - self.reverse_sort_zenodo_data_versioned_dois() - self.generate_files() - if True not in [release["citability"] == "full" for release in self.releases]: - self.message = "no valid CITATION.cff found in any release." - return - self.determine_latest_schema_dot_org() - self.message = "OK" - - def determine_latest_schema_dot_org(self): - for release in self.releases: - if "schema_dot_org" in release["files"].keys(): - self.latest_schema_dot_org = release["files"]["schema_dot_org"] - return self - - def filter_zenodo_data_versioned_dois(self): - def select_github_url(): - github_link = None - for identifier in hit["metadata"]["related_identifiers"]: - if identifier["scheme"] == "url" and \ - identifier["relation"] == "isSupplementTo" and \ - identifier["identifier"].startswith("https://github.com/"): - github_link = identifier["identifier"] - else: - # Associated Zenodo record does not contain a related identifier with "isSupplementTo" relation - # that points to GitHub. Continue with the next related identifier: - continue - return github_link - - hits = self.zenodo_data["versioned_dois"]["hits"]["hits"] - # already include the "files" key here, even though we're only filling it later: - files = dict() - for hit in hits: - try: - doi = hit["metadata"]["doi"] - date_published = hit["metadata"]["publication_date"] - url = select_github_url() - if url is not None: - tag = re.sub("^.*(/tree/)", "", url) - self.releases.append({ - "citability": "doi-only", - "datePublished": date_published, - "doi": doi, - "files": files, - "tag": tag, - "url": url}) - except Exception as e: - print("Something unexpected happened when trying to retrieve versioned doi info" + - " but I'll try to continue. {0}".format(e)) - return self - - def fetch_zenodo_data_conceptdoi(self): - zenodo_id = self.doi.replace("10.5281/zenodo.", "") - url = "https://zenodo.org/api/records/" + zenodo_id - headers = { - 'Authorization': 'Bearer ' + os.environ.get('ZENODO_ACCESS_TOKEN') - } - r = requests.get(url, headers=headers) - r.raise_for_status() - self.zenodo_data["conceptdoi"] = r.json() - return self - - def fetch_zenodo_data_versioned_dois(self): - url = "https://zenodo.org/api/records?q=conceptdoi:\"{0}\"&all_versions=true&size=100".format(self.doi) - headers = { - 'Authorization': 'Bearer ' + os.environ.get('ZENODO_ACCESS_TOKEN') - } - r = requests.get(url, headers=headers) - r.raise_for_status() - self.zenodo_data["versioned_dois"] = r.json() - hits = self.zenodo_data["versioned_dois"]["hits"]["hits"] - hits_sorted = sorted(hits, key=lambda hit: hit["metadata"]["publication_date"]) - self.title = hits_sorted[-1]["metadata"]["title"] - return self - - def generate_files(self): - for release in self.releases: - try: - override = { - "doi": release["doi"], - "date-released": datetime.strptime(release["datePublished"], "%Y-%m-%d").date(), - "version": release["tag"] - } - remove = ["commit"] - citation = Citation(url=release["url"], override=override, remove=remove) - try: - release["files"] = dict({ - "bibtex": citation.as_bibtex(), - "codemeta": citation.as_codemeta(), - "cff": citation.cffstr, - "schema_dot_org": citation.as_schema_dot_org(), - "endnote": citation.as_enw(), - "ris": citation.as_ris() - }) - release["citability"] = "full" - except Exception: - continue - except Exception as e: - if str(e).startswith("Error requesting file: "): - continue - elif str(e).startswith("Provided CITATION.cff does not seem valid YAML."): - continue - elif type(e).__name__ == "ScannerError": - continue - else: - print("Something went wrong while generating a file for a release:") - print(e) - print("Continuing") - continue - return self - - def is_concept_doi(self): - return "conceptdoi" in self.zenodo_data["conceptdoi"] and \ - self.doi == self.zenodo_data["conceptdoi"]["conceptdoi"] - - def is_zenodo_doi(self): - return self.doi.startswith("10.5281/zenodo.") - - def reverse_sort_zenodo_data_versioned_dois(self): - self.releases.sort(key=lambda release: release["tag"], reverse=True) - return self - - -def get_citations(dois_data): - n_dois = len(dois_data) - - for i_dois, data in enumerate(dois_data): - try: - release_id = data["release_id"] - #first create the entry in the mention table if it doesn't exist yet: - if release_id is None: - backend_release_response = requests.post('{}/release'.format(backend_url), '{{"software": "{}"}}'.format(data["software_id"]), headers={'Authorization': 'Bearer ' + jwt_token, 'Prefer': 'return=representation'}) - release_id = backend_release_response.json()[0]["id"] - - #now set the releases_scraped_at so the release gets pushed back in the scraping order - requests.patch('{}/release?id=eq.{}'.format(backend_url,release_id), '{{"releases_scraped_at": "{}"}}'.format(datetime.now()), headers={'Authorization': 'Bearer ' + jwt_token}) - - doi = data["concept_doi"] - release = ReleaseScraper(doi) - if release.is_citable: - document = { - "is_citable": release.is_citable, - "latest_schema_dot_org": release.latest_schema_dot_org - } - backend_release_response = requests.patch('{}/release?id=eq.{}'.format(backend_url,release_id), document, headers={'Authorization': 'Bearer ' + jwt_token}) - try: - backend_release_response.raise_for_status() - except requests.HTTPError as e: - print(e.response.text) - raise e - if release_id is None: - release_id = backend_release_response.json()[0]["id"] - - release_data = release.releases - release_data_to_save = list() - for release_entry in release_data: - release_content = { - "release_id": release_id, - "citability": release_entry["citability"], - "date_published": release_entry["datePublished"], - "doi": release_entry["doi"], - "tag": release_entry["tag"], - "url": release_entry["url"] - } - if release_entry["citability"] == "full": - release_content["bibtex"] = release_entry["files"]["bibtex"], - release_content["cff"] = release_entry["files"]["cff"], - release_content["codemeta"] = release_entry["files"]["codemeta"], - release_content["endnote"] = release_entry["files"]["endnote"], - release_content["ris"] = release_entry["files"]["ris"], - release_content["schema_dot_org"] = release_entry["files"]["schema_dot_org"] - else: - release_content["bibtex"] = None, - release_content["cff"] = None, - release_content["codemeta"] = None, - release_content["endnote"] = None, - release_content["ris"] = None, - release_content["schema_dot_org"] = None - - release_data_to_save.append(release_content) - - backend_release_content_response = requests.post('{}/release_content?on_conflict=doi'.format(backend_url), json=release_data_to_save, headers={'Authorization': 'Bearer ' + jwt_token, 'Prefer': 'resolution=merge-duplicates'}) - try: - backend_release_content_response.raise_for_status() - except requests.HTTPError as e: - print(e.response.text) - raise e - - if release.message == "OK": - print("{0}/{1} \"{2}\" ({3}): {4}".format(i_dois + 1, n_dois, doi, - release.title, release.message)) - else: - print('{0}/{1}: {2} {3}'.format(i_dois + 1, n_dois, doi, - release.message)) - except BaseException as e: - print("Something went wrong with getting the releases for software with slug " + data["slug"] + ":") - print(e) - print("Continuing with the next release") - - -if __name__ == "__main__": - print("Start scraping releases") - backend_url = os.environ.get('POSTGREST_URL') - number_releases_to_scrape = os.environ.get('MAX_REQUESTS_GITHUB', default='6') - jwt_secret = os.environ.get('PGRST_JWT_SECRET') - jwt_token = jwt.encode({"role": "rsd_admin", "exp": datetime.now() + timedelta(minutes = 10)}, jwt_secret, algorithm="HS256") - response_dois = requests.get('{}/rpc/software_join_release?concept_doi=not.is.null&order=releases_scraped_at.nullsfirst&limit={}'.format(backend_url, number_releases_to_scrape), headers={'Authorization': 'Bearer ' + jwt_token}) - dois_data = response_dois.json() - github_api_token = os.environ.get('API_CREDENTIALS_GITHUB') - get_citations(dois_data) - print("Done scraping releases") diff --git a/scrapers/src/main/resources/requirements.txt b/scrapers/src/main/resources/requirements.txt deleted file mode 100644 index 90f97bb..0000000 --- a/scrapers/src/main/resources/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) -# SPDX-FileCopyrightText: 2022 Netherlands eScience Center -# -# SPDX-License-Identifier: Apache-2.0 - -beautifulsoup4==4.6.0 -requests>=2.21.0 -python-dateutil==2.6.1 -Pyzotero==1.5.2 -cffconvert==1.3.3 -PyJWT==2.3.0 diff --git a/scrapers/src/main/resources/zotero.py b/scrapers/src/main/resources/zotero.py deleted file mode 100644 index a331ee8..0000000 --- a/scrapers/src/main/resources/zotero.py +++ /dev/null @@ -1,187 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) -# SPDX-FileCopyrightText: 2022 Netherlands eScience Center -# -# SPDX-License-Identifier: Apache-2.0 - -""" -https://pyzotero.readthedocs.io/en/latest/#retrieving-version-information - -""" - -import logging -from dateutil.parser import parse -import requests -from bs4 import BeautifulSoup -from pyzotero import zotero -import os -import jwt -from datetime import datetime, timedelta - - -def get_project_keys(client): - collections = client.everything(client.collections_top()) - projects_collection = [c for c in collections if c['data']['name'] == 'Projects'][0] - projects = client.everything(client.collections_sub(projects_collection['key'])) - return list(map(lambda x: x['key'], projects)) - - -def get_last_version(): - last_version_item = requests.get('{}/mention?select=version&order=version.desc&limit=1'.format(backend_url), headers={'Authorization': 'Bearer ' + jwt_token}).json() - return 0 if len(last_version_item) == 0 else last_version_item[0]['version'] - - -# DOIs can be either an URL or just the identifier, so make sure its a URL -def doi_to_url(doi): - if doi[:4] == 'http': - return doi - else: - return 'https://doi.org/' + doi - - -# URL is constructed from doi (either as doi field or in 'extra'), or else from the 'url' field -def get_url_for_zotero_item(item): - if 'DOI' in item['data'] and item['data']['DOI']: - return doi_to_url(item['data']['DOI']) - else: - for extra in item['data']['extra'].split('\n'): - if extra[:3] == 'doi' or extra[:3] == 'DOI': - return doi_to_url(extra[5:]) - if 'url' in item['data'] and item['data']['url']: - return item['data']['url'] - return None - - -def get_blog_fields(zotero_item): - try: - data = requests.get(zotero_item['data']['url']).text - soup = BeautifulSoup(data, 'html.parser') - authors = soup.find_all('meta', attrs={'name': 'author'}) - author = authors[0].attrs["content"] if len(authors) > 0 else None - image = soup.find("meta", property="og:image").attrs["content"] - return author, image - except: - return None, None - - -def get_mentions(since_version=None, keys_data=None): - by_version = since_version is not None - by_key = keys_data is not None - - if by_version and by_key: - raise Exception("Use either 'since_version' or 'keys_data', not both") - - client = zotero.Zotero(zotero_library, 'group', zotero_api_token) - - if by_key: - keys = ",".join([k["zotero_key"] for k in keys_data]) - items = client.everything(client.items(itemKey=keys)) - print('Found %d items in Zotero library %s based on supplied key(s).' % (len(items), - zotero_library)) - else: - their_last_version = client.last_modified_version() - our_last_version = get_last_version() - print(('Database collection \'mention\' is currently at version %d; Zotero library %s is cur' + - 'rently at version %d.') % (our_last_version, zotero_library, their_last_version)) - if since_version is None: - since_version = our_last_version - items = (client.everything(client.items(since=since_version))) - - print('Found %d new or updated items in Zotero library %s since version %d.' % (len(items), - zotero_library, - since_version)) - - items_to_save = [] - - project_keys = get_project_keys(client) - supported_types = ['attachment', 'blogPost', 'book', 'bookSection', 'computerProgram', 'conferencePaper', - 'document', 'interview', 'journalArticle', 'magazineArticle', 'manuscript', 'newspaperArticle', - 'note', 'presentation', 'radioBroadcast', 'report', 'thesis', 'videoRecording', 'webpage'] - - n_items = len(items) - for item_index, item in enumerate(items): - - if 'title' not in item['data'] or not item['data']['title']: - print("{0}/{1}: {2} does not have a title." - .format(item_index+1, n_items, item['key'])) - continue - item_collection_keys = item['data'].get('collections', []) - if len(set.intersection(set(item_collection_keys), set(project_keys))) == 0: - print("{0}/{1}: {2} is not part of a project ({3})." - .format(item_index + 1, n_items, item['key'], item["data"]["title"])) - continue - - try: - item_date = parse(item['data']['date']).isoformat()[:19] + 'Z' - except: - print("{0}/{1}: {2} has a date problem ({3})." - .format(item_index + 1, n_items, item['key'], item["data"]["title"])) - continue - - if item["data"]["itemType"] not in supported_types: - print("{0}/{1}: {2} not a supported type ({3})." - .format(item_index + 1, n_items, item['key'], item["data"]["title"])) - continue - - # item is good as far as we know - to_save = { - 'version': item['version'], - 'title': item['data'].get('title', ''), - 'type': item['data']['itemType'], - 'zotero_key': item['key'], - 'is_featured': False, - 'date': item_date, - 'scraped_at': str(datetime.now()), - 'author': None, - 'image': None, - 'url': None - } - url = get_url_for_zotero_item(item) - if url: - to_save['url'] = url - - if item['data']['url'] and '://blog.esciencecenter.nl/' in item['data']['url']: - (author, image) = get_blog_fields(item) - if author is None: - print("{0}/{1}: {2} cannot scrape the author from blog.esciencecenter.nl ({3})." - .format(item_index + 1, n_items, item['key'], item['data']['url'])) - continue - if image is None: - print("{0}/{1}: {2} cannot scrape the image from blog.esciencecenter.nl ({3})." - .format(item_index + 1, n_items, item['key'], item['data']['url'])) - continue - - to_save['is_featured'] = True - to_save['author'] = author - to_save['image'] = image - - print("{0}/{1}: {2} is going to be added to the mentions collection." - .format(item_index + 1, n_items, item['key'])) - items_to_save.append(to_save) - - if len(items_to_save) > 0: - print("Items to save: %s" % len(items_to_save)) - resp = requests.post( - backend_url + '/mention?on_conflict=zotero_key', - json=items_to_save, - headers={'Authorization': 'Bearer %s' % jwt_token, 'Prefer': 'resolution=merge-duplicates'} - ) - resp.raise_for_status() - - if by_key: - ids_to_update_scraped_at = ",".join([k["id"] for k in keys_data]) - requests.patch( - backend_url + '/mention?id=in.(' + ids_to_update_scraped_at + ')', - json={'scraped_at': str(datetime.now())}, - headers={'Authorization': 'Bearer %s' % jwt_token} - ) - -if __name__ == "__main__": - print("Start scraping from Zotero") - backend_url = os.environ.get('POSTGREST_URL') - number_mentions_to_scrape = os.environ.get('MAX_REQUESTS_GITHUB', default='6') - jwt_secret = os.environ.get('PGRST_JWT_SECRET') - jwt_token = jwt.encode({"role": "rsd_admin", "exp": datetime.now() + timedelta(minutes = 10)}, jwt_secret, algorithm="HS256") - zotero_library = os.environ.get('ZOTERO_LIBRARY') - zotero_api_token = os.environ.get('ZOTERO_API_TOKEN') - get_mentions() - print("Done scraping from Zotero") diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/BibtexParserTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/BibtexParserTest.java index 13e74c2..79d795f 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/BibtexParserTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/BibtexParserTest.java @@ -1,86 +1,89 @@ -// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper; -import nl.esciencecenter.rsd.scraper.doi.ExternalMentionRecord; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - import java.io.IOException; import java.io.StringReader; import java.util.Map; +import nl.esciencecenter.rsd.scraper.doi.ExternalMentionRecord; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; class BibtexParserTest { @Test void givenValidBibtex_whenParsing_thenParsedCorrectly() throws InvalidInputException, IOException { String validBibtex = """ - @misc{Brughmans2018, - author = {Brughmans, Tom}, - title = {{Network structures and assembling code in Netlogo, Tutorial}}, - url = {https://archaeologicalnetworks.wordpress.com/resources/#structures}, - year = {2018} - } - @article{Brughmans2016c, - author = {Brughmans, Tom and Poblome, Jeroen}, - doi = {10.15184/aqy.2016.35}, - file = {:Users/au656977/Library/Application Support/Mendeley Desktop/Downloaded/Brughmans, Poblome - 2016 - Roman bazaar or market economy Explaining tableware distributions through computational modelling.pdf:pdf}, - journal = {Antiquity}, - mendeley-groups = {Brughmans,PNAS,old/ERC_StG2019_B2,old/ERC_StG2019_B1,Roman-simulations,TO ADD,TRAJ manifesto,ERCStG2021_B2}, - number = {350}, - pages = {393--408}, - title = {{Roman bazaar or market economy? Explaining tableware distributions through computational modelling}}, - volume = {90}, - year = {2016} - } - @article{Brughmans2016d, - author = {Brughmans, Tom and Poblome, Jeroen}, - file = {:Users/au656977/Library/Application Support/Mendeley Desktop/Downloaded/Brughmans, Poblome - 2016 - MERCURY an agent-based model of tableware trade in the Roman East.pdf:pdf}, - journal = {Journal of Artificial Societies and Social Simulation}, - mendeley-groups = {Brughmans,old/ERC_StG2019_B2,Roman-simulations,TO ADD,TRAJ manifesto}, - number = {1}, - pages = {http://jasss.soc.surrey.ac.uk/19/1/3.html}, - title = {{MERCURY: an agent-based model of tableware trade in the Roman East}}, - volume = {19}, - year = {2016} - } - """; + @misc{Brughmans2018, + author = {Brughmans, Tom}, + title = {{Network structures and assembling code in Netlogo, Tutorial}}, + url = {https://archaeologicalnetworks.wordpress.com/resources/#structures}, + year = {2018} + } + @article{Brughmans2016c, + author = {Brughmans, Tom and Poblome, Jeroen}, + doi = {10.15184/aqy.2016.35}, + file = {:Users/au656977/Library/Application Support/Mendeley Desktop/Downloaded/Brughmans, Poblome - 2016 - Roman bazaar or market economy Explaining tableware distributions through computational modelling.pdf:pdf}, + journal = {Antiquity}, + mendeley-groups = {Brughmans,PNAS,old/ERC_StG2019_B2,old/ERC_StG2019_B1,Roman-simulations,TO ADD,TRAJ manifesto,ERCStG2021_B2}, + number = {350}, + pages = {393--408}, + title = {{Roman bazaar or market economy? Explaining tableware distributions through computational modelling}}, + volume = {90}, + year = {2016} + } + @article{Brughmans2016d, + author = {Brughmans, Tom and Poblome, Jeroen}, + file = {:Users/au656977/Library/Application Support/Mendeley Desktop/Downloaded/Brughmans, Poblome - 2016 - MERCURY an agent-based model of tableware trade in the Roman East.pdf:pdf}, + journal = {Journal of Artificial Societies and Social Simulation}, + mendeley-groups = {Brughmans,old/ERC_StG2019_B2,Roman-simulations,TO ADD,TRAJ manifesto}, + number = {1}, + pages = {http://jasss.soc.surrey.ac.uk/19/1/3.html}, + title = {{MERCURY: an agent-based model of tableware trade in the Roman East}}, + volume = {19}, + year = {2016} + } + """; Map result = BibtexParser.parse(new StringReader(validBibtex)); Assertions.assertEquals(3, result.size()); Assertions.assertEquals("Brughmans, Tom", result.get("Brughmans2018").authors()); - Assertions.assertEquals("Network structures and assembling code in Netlogo, Tutorial", result.get("Brughmans2018") - .title()); + Assertions.assertEquals( + "Network structures and assembling code in Netlogo, Tutorial", + result.get("Brughmans2018").title() + ); Assertions.assertEquals(2018, result.get("Brughmans2018").publicationYear()); } @Test void givenValidBibtex_whenParsing_thenParsedCorrectly2() throws InvalidInputException, IOException { String validBibtex = """ - @book{romanowska_agent-based_2021, - edition = {Electronic}, - title = {Agent-{Based} {Modeling} for {Archaeology}}, - isbn = {978-1-947864-38-2}, - url = {https://www.sfipress.org/books/agent-based-modeling-archaeology}, - abstract = {To fully understand not only the past, but also the trajectories, of human societies, we need a more dynamic view of human social systems. Agent-based modeling (ABM), which can create fine-scale models of behavior over time and space, may reveal important, general patterns of human activity. Agent-Based Modeling for Archaeology is the first ABM textbook designed for researchers studying the human past. Appropriate for scholars from archaeology, the digital humanities, and other social sciences, this book offers novices and more experienced ABM researchers a modular approach to learning ABM and using it effectively. Readers will find the necessary background, discussion of modeling techniques and traps, references, and algorithms to use ABM in their own work. They will also find engaging examples of how other scholars have applied ABM, ranging from the study of the intercontinental migration pathways of early hominins, to the weather–crop–population cycles of the American Southwest, to the trade networks of Ancient Rome. This textbook provides the foundations needed to simulate the complexity of past human societies, offering researchers a richer understanding of the past—and likely future—of our species.}, - urldate = {2022-02-01}, - publisher = {SFI Press}, - author = {Romanowska, Iza}, - month = aug, - year = {2021}, - doi = {10.37911/9781947864382}, - }"""; + @book{romanowska_agent-based_2021, + edition = {Electronic}, + title = {Agent-{Based} {Modeling} for {Archaeology}}, + isbn = {978-1-947864-38-2}, + url = {https://www.sfipress.org/books/agent-based-modeling-archaeology}, + abstract = {To fully understand not only the past, but also the trajectories, of human societies, we need a more dynamic view of human social systems. Agent-based modeling (ABM), which can create fine-scale models of behavior over time and space, may reveal important, general patterns of human activity. Agent-Based Modeling for Archaeology is the first ABM textbook designed for researchers studying the human past. Appropriate for scholars from archaeology, the digital humanities, and other social sciences, this book offers novices and more experienced ABM researchers a modular approach to learning ABM and using it effectively. Readers will find the necessary background, discussion of modeling techniques and traps, references, and algorithms to use ABM in their own work. They will also find engaging examples of how other scholars have applied ABM, ranging from the study of the intercontinental migration pathways of early hominins, to the weather–crop–population cycles of the American Southwest, to the trade networks of Ancient Rome. This textbook provides the foundations needed to simulate the complexity of past human societies, offering researchers a richer understanding of the past—and likely future—of our species.}, + urldate = {2022-02-01}, + publisher = {SFI Press}, + author = {Romanowska, Iza}, + month = aug, + year = {2021}, + doi = {10.37911/9781947864382}, + }"""; Map result = BibtexParser.parse(new StringReader(validBibtex)); Assertions.assertEquals(1, result.size()); Assertions.assertEquals("Romanowska, Iza", result.get("romanowska_agent-based_2021").authors()); - Assertions.assertEquals("Agent-Based Modeling for Archaeology", result.get("romanowska_agent-based_2021") - .title()); + Assertions.assertEquals( + "Agent-Based Modeling for Archaeology", + result.get("romanowska_agent-based_2021").title() + ); Assertions.assertEquals("10.37911/9781947864382", result.get("romanowska_agent-based_2021").doi().toString()); Assertions.assertEquals(2021, result.get("romanowska_agent-based_2021").publicationYear()); } diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ThrottlerTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ThrottlerTest.java new file mode 100644 index 0000000..b4eda77 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/ThrottlerTest.java @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +class ThrottlerTest { + + @Test + void givenThrottlerWithFixedDelay_whenGettingTwoPermission_thenMinimumTimeRespected() throws InterruptedException { + int millisecondsBetweenPermissions = 100; + Throttler throttler = new Throttler(1, millisecondsBetweenPermissions, TimeUnit.MILLISECONDS); + + long tik = System.nanoTime(); + throttler.awaitPermission(); + throttler.awaitPermission(); + long tok = System.nanoTime(); + + Assertions.assertTrue((tok - tik) / 1_000_000 >= millisecondsBetweenPermissions); + } + + @Test + void givenThrottlerWithRandomDelay_whenGettingTwoPermission_thenMinimumTimeRespected() throws InterruptedException { + int min = 10; + int max = 500; + Random random = new Random(); + + for (int i = 0; i < 10; i++) { + int millisecondsBetweenPermissions = random.nextInt(max - min) + min; + Throttler throttler = new Throttler(1, millisecondsBetweenPermissions, TimeUnit.MILLISECONDS); + + long tik = System.nanoTime(); + throttler.awaitPermission(); + throttler.awaitPermission(); + long tok = System.nanoTime(); + + Assertions.assertTrue((tok - tik) / 1_000_000 >= millisecondsBetweenPermissions); + } + } + + @Disabled("This is for playing around with the Throttler class using prints, not for automated testing.") + @Test + void testThrottlerFuturesWithPrintStatements() { + Throttler throttler = new Throttler(3, 1000, TimeUnit.MILLISECONDS); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + + List> futures = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + int finalI = i; + Future task = throttler.runThrottledTask(() -> finalI, executorService); + futures.add(task); + } + + futures.forEach(f -> { + try { + System.out.println(f.get()); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsIT.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsIT.java index 864378f..1ba4b37 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsIT.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsIT.java @@ -27,9 +27,7 @@ void getWithoutToken() { @Test void getWrongUri() { final String wrongUri = gitHubApiUrl + "/repos/research-software-directory/wrongRepo"; - Exception thrown = Assertions.assertThrows( - Exception.class, () -> Utils.get(wrongUri) - ); + Exception thrown = Assertions.assertThrows(Exception.class, () -> Utils.get(wrongUri)); Assertions.assertTrue(thrown.getMessage().startsWith("Error fetching data")); } diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsTest.java index 2d5be15..293720a 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/UtilsTest.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2026 Netherlands eScience Center // SPDX-FileCopyrightText: 2022 Christian Meeßen (GFZ) // SPDX-FileCopyrightText: 2022 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences // @@ -9,9 +9,68 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; class UtilsTest { + @ParameterizedTest + @CsvSource( + value = { + "abc,abc", + " abc,abc", + " abc ,abc", + " abc,abc", + "a bc,a bc", + "a b c,a b c", + " a b c,a b c", + "a b,a b", + " a b ,a b", + }, + ignoreLeadingAndTrailingWhitespace = false + ) + void givenString_whenSanitised_thenLeadingAndTrailingAndDuplicateSpacesRemoved( + String input, + String expectedOutput + ) { + Assertions.assertEquals(expectedOutput, Utils.sanitiseWhitespace(input)); + } + + @Test + void givenNull_whenSanitised_thenNullReturned() { + Assertions.assertNull(Utils.sanitiseWhitespace(null)); + } + + @ParameterizedTest + @CsvSource( + value = { + "abc,abc", + "ABC,abc", + "123,123", + "09AbC123,09abc123", + " abc,abc", + " abc ,abc", + " abc,abc", + "a bc,a-bc", + "a b c,a-b-c", + " a b c,a-b-c", + "a b,a-b", + " a b ,a-b", + "a_b,a-b", + "a@b,a-b", + "a!@#$%^&*()_b,a-b", + "aBc D,abc-d", + "a-b!@#$%^&,a-b", + "!@#$%^&a-b,a-b", + "!@#$%^&a-b!@#$%^&,a-b", + "---AB----,ab", + }, + ignoreLeadingAndTrailingWhitespace = false + ) + void givenString_whenSlugifies_thenCorrectSlugReturned(String input, String expectedOutput) { + Assertions.assertEquals(expectedOutput, Utils.slugify(input)); + } + @Test void urlEncode() { Assertions.assertEquals("one%2Ftwo", Utils.urlEncode("one/two")); diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DataCiteConnectorTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DataCiteConnectorTest.java new file mode 100644 index 0000000..6344e7f --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DataCiteConnectorTest.java @@ -0,0 +1,265 @@ +// SPDX-FileCopyrightText: 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2026 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.doi; + +import java.time.ZonedDateTime; +import java.util.Collection; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class DataCiteConnectorTest { + + @Test + void givenDoiResponse_whenParsingVersionDois_thenVersionDoisReturned() { + // from https://api.datacite.org/dois/10.5281/zenodo.6379973, a large portion was left out + // editorconfig-checker-disable + String response = """ + { + "data": { + "id": "10.5281/zenodo.6379973", + "type": "dois", + "relationships": { + "versions": { + "data": [ + { + "id": "10.5281/zenodo.6379974", + "type": "dois" + }, + { + "id": "10.5281/zenodo.6782555", + "type": "dois" + } + ] + }, + "versionOf": { + "data": [ + { + "id": "10.5281/zenodo.6379973", + "type": "dois" + } + ] + } + } + } + }"""; + // editorconfig-checker-enable + + Collection versionDois = DataciteConnector.DataciteParser.parseVersionDois(response); + + Assertions.assertEquals(2, versionDois.size()); + Assertions.assertTrue(versionDois.contains(Doi.fromString("10.5281/zenodo.6379974"))); + Assertions.assertTrue(versionDois.contains(Doi.fromString("10.5281/zenodo.6782555"))); + } + + @Test + void givenDoiResponse_whenParsingAsMention_thenMentionReturned() { + // from https://api.datacite.org/dois/10.5281/zenodo.6379973, some data was left out + // editorconfig-checker-disable + String response = """ + { + "data": { + "id": "10.5281/zenodo.3877116", + "type": "dois", + "attributes": { + "doi": "10.5281/zenodo.3877116", + "prefix": "10.5281", + "suffix": "zenodo.3877116", + "identifiers": [ + { + "identifier": "https://zenodo.org/record/5898417", + "identifierType": "URL" + } + ], + "alternateIdentifiers": [ + { + "alternateIdentifierType": "URL", + "alternateIdentifier": "https://zenodo.org/record/5898417" + } + ], + "creators": [ + { + "name": "Hidding, Johan", + "givenName": "Johan", + "familyName": "Hidding", + "affiliation": [ + "Netherlands eScience Center" + ], + "nameIdentifiers": [ + { + "schemeUri": "https://orcid.org", + "nameIdentifier": "https://orcid.org/0000-0002-7550-1796", + "nameIdentifierScheme": "ORCID" + } + ] + } + ], + "titles": [ + { + "title": "Entangled" + } + ], + "publisher": "Zenodo", + "container": {}, + "publicationYear": 2022, + "subjects": [ + { + "subject": "Literate Programming" + } + ], + "contributors": [], + "dates": [ + { + "date": "2022-01-24", + "dateType": "Issued" + } + ], + "language": null, + "types": { + "ris": "COMP", + "bibtex": "misc", + "citeproc": "article", + "schemaOrg": "SoftwareSourceCode", + "resourceTypeGeneral": "Software" + }, + "relatedIdentifiers": [ + { + "relationType": "IsSupplementTo", + "relatedIdentifier": "https://github.com/entangled/entangled/tree/v1.3.0", + "relatedIdentifierType": "URL" + }, + { + "relationType": "HasVersion", + "relatedIdentifier": "10.5281/zenodo.3877117", + "relatedIdentifierType": "DOI" + }, + { + "relationType": "HasVersion", + "relatedIdentifier": "10.5281/zenodo.3885761", + "relatedIdentifierType": "DOI" + }, + { + "relationType": "HasVersion", + "relatedIdentifier": "10.5281/zenodo.5898417", + "relatedIdentifierType": "DOI" + } + ], + "relatedItems": [], + "sizes": [], + "formats": [], + "version": "v1.3.0", + "rightsList": [ + { + "rights": "Apache License 2.0", + "rightsUri": "http://www.apache.org/licenses/LICENSE-2.0", + "schemeUri": "https://spdx.org/licenses/", + "rightsIdentifier": "apache-2.0", + "rightsIdentifierScheme": "SPDX" + }, + { + "rights": "Open Access", + "rightsUri": "info:eu-repo/semantics/openAccess" + } + ], + "descriptions": [ + { + "description": "Literate programming is a programming paradigm introduced by Donald Knuth in which a program is given as an explanation of the program logic in a natural language, such as English, interspersed with snippets of macros and traditional source code, from which a compilable source code can be generated. Entangled makes writing literate programs easier by keeping code blocks in Markdown up-to-date with generated source files. By monitoring the tangled source files, any change in the master document or source files is reflected in the other.", + "descriptionType": "Abstract" + } + ], + "geoLocations": [], + "fundingReferences": [], + "url": "https://zenodo.org/record/3877116", + "contentUrl": null, + "metadataVersion": 2, + "schemaVersion": "http://datacite.org/schema/kernel-4", + "source": "mds", + "isActive": true, + "state": "findable", + "reason": null, + "viewCount": 0, + "viewsOverTime": [], + "downloadCount": 0, + "downloadsOverTime": [], + "referenceCount": 0, + "citationCount": 0, + "citationsOverTime": [], + "partCount": 0, + "partOfCount": 0, + "versionCount": 3, + "versionOfCount": 0, + "created": "2020-06-04T17:15:18.000Z", + "registered": "2020-06-04T17:15:19.000Z", + "published": "2022", + "updated": "2022-01-24T15:40:07.000Z" + }, + "relationships": { + "client": { + "data": { + "id": "cern.zenodo", + "type": "clients" + } + }, + "provider": { + "data": { + "id": "cern", + "type": "providers" + } + }, + "media": { + "data": { + "id": "10.5281/zenodo.3877116", + "type": "media" + } + }, + "references": { + "data": [] + }, + "citations": { + "data": [] + }, + "parts": { + "data": [] + }, + "partOf": { + "data": [] + }, + "versions": { + "data": [ + { + "id": "10.5281/zenodo.5898417", + "type": "dois" + }, + { + "id": "10.5281/zenodo.3885761", + "type": "dois" + }, + { + "id": "10.5281/zenodo.3877117", + "type": "dois" + } + ] + }, + "versionOf": { + "data": [] + } + } + } + }"""; + // editorconfig-checker-enable + + ExternalMentionRecord mention = Assertions.assertDoesNotThrow(() -> + DataciteConnector.DataciteParser.parseMention(response) + ); + + Assertions.assertEquals(Doi.fromString("10.5281/zenodo.3877116"), mention.doi()); + Assertions.assertEquals("Entangled", mention.title()); + Assertions.assertEquals("Johan Hidding", mention.authors()); + Assertions.assertEquals("DataCite", mention.source()); + Assertions.assertEquals("v1.3.0", mention.version()); // from a related Git URL + Assertions.assertEquals(2022, mention.publicationYear()); + Assertions.assertEquals(ZonedDateTime.parse("2020-06-04T17:15:19.000Z"), mention.doiRegistrationDate()); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepositoryTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepositoryTest.java deleted file mode 100644 index 9a814de..0000000 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepositoryTest.java +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center -// -// SPDX-License-Identifier: Apache-2.0 - -package nl.esciencecenter.rsd.scraper.doi; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.Collection; -import java.util.List; - -class DataciteMentionRepositoryTest { - - @Test - void givenCollectionOfStrings_whenJoining_thenCorrectStringReturned() { - Doi doi1 = Doi.fromString("10.000/1"); - Doi doi2 = Doi.fromString("10.2/2"); - Doi doi3 = Doi.fromString("10.3/abc-def"); - Collection strings = List.of(doi1, doi2, doi3); - - String joinedString = DataciteMentionRepository.joinDoisForGraphqlQuery(strings); - - String expected = "\"%s\",\"%s\",\"%s\"".formatted(doi1, doi2, doi3); - Assertions.assertEquals(expected, joinedString); - } -} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DoiTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DoiTest.java index 58ff894..3397668 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DoiTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/DoiTest.java @@ -13,7 +13,8 @@ class DoiTest { @ParameterizedTest - @ValueSource(strings = { + @ValueSource( + strings = { "10.2533/chimia.2024.525", "10.1017/9781108881425", "10.3390/photonics11070630", @@ -21,14 +22,16 @@ class DoiTest { "10.1007/978-3-030-83508-8_2", "10.22541/essoar.171500959.99365288/v1", "10.1016/j.eswa.2023.120561", - }) + } + ) void givenValidDoi_whenInstanceCreated_thenNoExceptionThrown(String validDoi) { Doi doi = Assertions.assertDoesNotThrow(() -> Doi.fromString(validDoi)); Assertions.assertNotNull(doi); } @ParameterizedTest - @ValueSource(strings = { + @ValueSource( + strings = { "10.2533", "10.2533/", "https://doi.org/10.2533/chimia.2024.525", @@ -36,7 +39,8 @@ void givenValidDoi_whenInstanceCreated_thenNoExceptionThrown(String validDoi) { "10.3390/photonics11070630 ", "11.1016/j.eswa.2023.120561", "", - }) + } + ) void givenInValidDoi_whenCreatingInstance_thenExceptionThrown(String invalidDoi) { Assertions.assertThrows(RuntimeException.class, () -> Doi.fromString(invalidDoi)); } diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/MainMentionsTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/MainMentionsTest.java index 3eb6e1b..91298c3 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/MainMentionsTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/MainMentionsTest.java @@ -1,48 +1,46 @@ -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.doi; +import java.util.Map; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.Map; - class MainMentionsTest { - @Test void givenValidDoiSourceData_whenParsing_thenMapReturned() { // editorconfig-checker-disable String validDoiSourceData = """ - [ - { - "DOI": "10.5240/B1FA-0EEC-C316-3316-3A73-L", - "RA": "EIDR" - }, - { - "DOI": "notADoi", - "status": "Invalid DOI" - }, - { - "DOI": "10.5281/zenodo.1436372", - "RA": "DataCite" - }, - { - "DOI": "10.5281/zenodo.2633819", - "RA": "DataCite" - }, - { - "DOI": "10.5281/zenodo.5825192", - "RA": "DataCite" - }, - { - "DOI": "10.35802/218300", - "RA": "Crossref" - } - ]"""; + [ + { + "DOI": "10.5240/B1FA-0EEC-C316-3316-3A73-L", + "RA": "EIDR" + }, + { + "DOI": "notADoi", + "status": "Invalid DOI" + }, + { + "DOI": "10.5281/zenodo.1436372", + "RA": "DataCite" + }, + { + "DOI": "10.5281/zenodo.2633819", + "RA": "DataCite" + }, + { + "DOI": "10.5281/zenodo.5825192", + "RA": "DataCite" + }, + { + "DOI": "10.35802/218300", + "RA": "Crossref" + } + ]"""; // editorconfig-checker-enable Map doiToSource = MainMentions.parseJsonDoiSources(validDoiSourceData); diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnectorTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnectorTest.java index fab34b5..cc916ec 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnectorTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenAlexConnectorTest.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 @@ -7,12 +7,11 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - import java.net.URI; import java.util.Collection; import java.util.Collections; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; class OpenAlexConnectorTest { @@ -34,10 +33,29 @@ void givenLocationWithBackSlashes_whenExtractedAsLocation_thenSlashesUrlEncoded( void givenEmptyCollection_whenGettingData_thenEmptyCollectionReturned() { OpenAlexConnector openAlexConnector = new OpenAlexConnector(); - Collection doiMentions = Assertions.assertDoesNotThrow(() -> openAlexConnector.mentionDataByDois(Collections.emptyList(), null)); + Collection doiMentions = Assertions.assertDoesNotThrow(() -> + openAlexConnector.mentionDataByDois(Collections.emptyList()) + ); Assertions.assertTrue(doiMentions.isEmpty()); - Collection openalexMentions = Assertions.assertDoesNotThrow(() -> openAlexConnector.mentionDataByOpenalexIds(Collections.emptyList(), null)); + Collection openalexMentions = Assertions.assertDoesNotThrow(() -> + openAlexConnector.mentionDataByOpenalexIds(Collections.emptyList()) + ); Assertions.assertTrue(openalexMentions.isEmpty()); } + + @Test + void givenOpenAlexId_whenConvertingToCitationsUrl_thenCorrectUrlReturned() { + String openAlexKey = "W12345"; + OpenalexId openalexId = OpenalexId.fromString("https://openalex.org/" + openAlexKey); + + String citationsUrl = OpenAlexConnector.citationsUri(openalexId); + + Assertions.assertEquals("https://api.openalex.org/works?filter=cites:" + openAlexKey, citationsUrl); + } + + @Test + void givenNullOpenAlexId_whenConvertingToCitationsUrl_thenNullPointerThrown() { + Assertions.assertThrowsExactly(NullPointerException.class, () -> OpenAlexConnector.citationsUri(null)); + } } diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenalexIdTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenalexIdTest.java index 0b20e9a..84cd237 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenalexIdTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/doi/OpenalexIdTest.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 @@ -13,25 +13,29 @@ class OpenalexIdTest { @ParameterizedTest - @ValueSource(strings = { + @ValueSource( + strings = { "https://openalex.org/W3160330321", "https://openalex.org/w3160330321", "https://openalex.org/W152867311", - }) + } + ) void givenValidOpenalexId_whenInstanceCreated_thenNoExceptionThrown(String validId) { OpenalexId openalexId = Assertions.assertDoesNotThrow(() -> OpenalexId.fromString(validId)); Assertions.assertNotNull(openalexId); } @ParameterizedTest - @ValueSource(strings = { + @ValueSource( + strings = { "http://openalex.org/W3160330321", "https://openalex.org/3160330321", "https://openalex.org/W3160330321/", "https://openalex.org/works/W3160330321", "W3160330321", "", - }) + } + ) void givenInValidOpenalexId_whenCreatingInstance_thenExceptionThrown(String invalidId) { Assertions.assertThrows(RuntimeException.class, () -> OpenalexId.fromString(invalidId)); } diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/CodebergScraperTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/CodebergScraperTest.java new file mode 100644 index 0000000..c3620a8 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/CodebergScraperTest.java @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: 2025 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2025 - 2026 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class CodebergScraperTest { + + @Test + void givenValidBasicDataJson_whenParsed_thenCorrectResultReturned() { + // editorconfig-checker-disable + String data = """ + { + "id": 43678, + "owner": { + "id": 48310, + "login": "privacyguides", + "login_name": "", + "source_id": 0, + "full_name": "Privacy Guides", + "email": "", + "avatar_url": "https://codeberg.org/avatars/ed2e53352d56adc4cf428df0f9e80913", + "html_url": "https://codeberg.org/privacyguides", + "language": "", + "is_admin": false, + "last_login": "0001-01-01T00:00:00Z", + "created": "2022-05-02T16:55:47+02:00", + "restricted": false, + "active": false, + "prohibit_login": false, + "location": "", + "pronouns": "", + "website": "https://privacyguides.org", + "description": "", + "visibility": "public", + "followers_count": 18, + "following_count": 0, + "starred_repos_count": 0, + "username": "privacyguides" + }, + "name": "privacyguides.org", + "full_name": "privacyguides/privacyguides.org", + "description": "Mirror of https://github.com/privacyguides/privacyguides.org - Protect your data against global mass surveillance programs.", + "empty": false, + "private": false, + "fork": false, + "template": false, + "parent": null, + "mirror": false, + "size": 159315, + "language": "Markdown", + "languages_url": "https://codeberg.org/api/v1/repos/privacyguides/privacyguides.org/languages", + "html_url": "https://codeberg.org/privacyguides/privacyguides.org", + "url": "https://codeberg.org/api/v1/repos/privacyguides/privacyguides.org", + "link": "", + "ssh_url": "ssh://git@codeberg.org/privacyguides/privacyguides.org.git", + "clone_url": "https://codeberg.org/privacyguides/privacyguides.org.git", + "original_url": "", + "website": "https://privacyguides.org", + "stars_count": 194, + "forks_count": 4, + "watchers_count": 18, + "open_issues_count": 0, + "open_pr_counter": 0, + "release_counter": 0, + "default_branch": "main", + "archived": false, + "created_at": "2022-05-02T17:34:40+02:00", + "updated_at": "2025-08-20T19:01:48+02:00", + "archived_at": "1970-01-01T01:00:00+01:00", + "permissions": { + "admin": false, + "push": false, + "pull": true + }, + "has_issues": true, + "external_tracker": { + "external_tracker_url": "https://github.com/privacyguides/privacyguides.org/issues", + "external_tracker_format": "https://github.com/privacyguides/privacyguides.org/issues/{index}", + "external_tracker_style": "", + "external_tracker_regexp_pattern": "" + }, + "has_wiki": false, + "wiki_branch": "master", + "globally_editable_wiki": false, + "has_pull_requests": true, + "has_projects": false, + "has_releases": false, + "has_packages": false, + "has_actions": false, + "ignore_whitespace_conflicts": false, + "allow_merge_commits": true, + "allow_rebase": true, + "allow_rebase_explicit": true, + "allow_squash_merge": true, + "allow_fast_forward_only_merge": false, + "allow_rebase_update": true, + "default_delete_branch_after_merge": false, + "default_merge_style": "squash", + "default_allow_maintainer_edit": false, + "default_update_style": "merge", + "avatar_url": "https://codeberg.org/repo-avatars/43678-266105cc5352d0f73a9c48abceab936b", + "internal": false, + "mirror_interval": "", + "object_format_name": "sha1", + "mirror_updated": "0001-01-01T00:00:00Z", + "repo_transfer": null, + "topics": [] + }"""; + // editorconfig-checker-enable + + BasicGitData parsedData = CodebergScraper.parseBasicData(data); + + Assertions.assertEquals(false, parsedData.archived()); + Assertions.assertNull(parsedData.license()); + Assertions.assertEquals(194, parsedData.starCount()); + Assertions.assertEquals(4, parsedData.forkCount()); + Assertions.assertEquals(0, parsedData.openIssueCount()); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeekTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeekTest.java index 56788b3..bffa4db 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeekTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/CommitsPerWeekTest.java @@ -11,13 +11,12 @@ import com.google.gson.GsonBuilder; import com.google.gson.JsonDeserializer; import com.google.gson.reflect.TypeToken; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - import java.time.Duration; import java.time.Instant; import java.time.Period; import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; class CommitsPerWeekTest { @@ -34,7 +33,6 @@ void givenInstance_whenValidOperations_thenCorrectResults() { Assertions.assertTrue(underlyingMap.containsKey(sundayMidnight1)); Assertions.assertEquals(10, underlyingMap.get(sundayMidnight1)); - commitsPerWeek.addCommits(sundayMidnight1, 20); underlyingMap = commitsPerWeek.getData(); @@ -42,8 +40,8 @@ void givenInstance_whenValidOperations_thenCorrectResults() { Assertions.assertTrue(underlyingMap.containsKey(sundayMidnight1)); Assertions.assertEquals(30, underlyingMap.get(sundayMidnight1)); - - Instant smallTimeAfterSundayMidnight1 = sundayMidnight1.plus(Duration.ofDays(3)) + Instant smallTimeAfterSundayMidnight1 = sundayMidnight1 + .plus(Duration.ofDays(3)) .plus(Duration.ofSeconds(12345)); commitsPerWeek.addCommits(smallTimeAfterSundayMidnight1, 10); @@ -52,7 +50,6 @@ void givenInstance_whenValidOperations_thenCorrectResults() { Assertions.assertTrue(underlyingMap.containsKey(sundayMidnight1)); Assertions.assertEquals(40, underlyingMap.get(sundayMidnight1)); - Instant sundayMidnight2 = sundayMidnight1.plus(Period.ofWeeks(5)); commitsPerWeek.addCommits(sundayMidnight2, 5); @@ -62,7 +59,6 @@ void givenInstance_whenValidOperations_thenCorrectResults() { Assertions.assertEquals(5, underlyingMap.get(sundayMidnight2)); Assertions.assertEquals(40, underlyingMap.get(sundayMidnight1)); - commitsPerWeek.addMissingZeros(); underlyingMap = commitsPerWeek.getData(); Assertions.assertEquals(6, underlyingMap.size()); @@ -72,16 +68,18 @@ void givenInstance_whenValidOperations_thenCorrectResults() { Assertions.assertEquals(0, underlyingMap.get(sundayMidnight1.plus(Period.ofWeeks(4)))); Assertions.assertEquals(40, underlyingMap.get(sundayMidnight1)); Assertions.assertEquals(5, underlyingMap.get(sundayMidnight2)); - } @Test void givenInstance_whenValidOperations_thenCorrectJsonProduced() { Gson gson = new GsonBuilder() - .registerTypeAdapter(Instant.class, (JsonDeserializer) (json, typeOfT, context) -> Instant.ofEpochSecond(Long.parseLong(json.getAsString()))) + .registerTypeAdapter( + Instant.class, + (JsonDeserializer) (json, typeOfT, context) -> + Instant.ofEpochSecond(Long.parseLong(json.getAsString())) + ) .create(); - TypeToken> mapTypeToken = new TypeToken<>() { - }; + TypeToken> mapTypeToken = new TypeToken<>() {}; CommitsPerWeek commitsPerWeek = new CommitsPerWeek(); Map shouldBeEmptyMap = gson.fromJson(commitsPerWeek.toJson(), mapTypeToken.getType()); @@ -91,22 +89,21 @@ void givenInstance_whenValidOperations_thenCorrectJsonProduced() { commitsPerWeek.addCommits(sundayMidnight1, 10); commitsPerWeek.addCommits(sundayMidnight1, 20); - Instant smallTimeAfterSundayMidnight1 = sundayMidnight1.plus(Duration.ofDays(3)) + Instant smallTimeAfterSundayMidnight1 = sundayMidnight1 + .plus(Duration.ofDays(3)) .plus(Duration.ofSeconds(12345)); commitsPerWeek.addCommits(smallTimeAfterSundayMidnight1, 10); Instant sundayMidnight2 = sundayMidnight1.plus(Period.ofWeeks(5)); commitsPerWeek.addCommits(sundayMidnight2, 5); - Map dataFromJson = gson.fromJson(commitsPerWeek.toJson(), mapTypeToken.getType()); Assertions.assertEquals(2, dataFromJson.size()); Assertions.assertEquals(40, dataFromJson.get(sundayMidnight1)); Assertions.assertEquals(5, dataFromJson.get(sundayMidnight2)); commitsPerWeek.addMissingZeros(); - dataFromJson = gson.fromJson(commitsPerWeek.toJson(), new TypeToken>() { - }.getType()); + dataFromJson = gson.fromJson(commitsPerWeek.toJson(), new TypeToken>() {}.getType()); Assertions.assertEquals(6, dataFromJson.size()); Assertions.assertEquals(0, dataFromJson.get(sundayMidnight1.plus(Period.ofWeeks(1)))); Assertions.assertEquals(0, dataFromJson.get(sundayMidnight1.plus(Period.ofWeeks(2)))); diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GithubScraperTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GithubScraperTest.java index 52af1d5..0ed6a87 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GithubScraperTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GithubScraperTest.java @@ -1,17 +1,19 @@ // SPDX-FileCopyrightText: 2022 - 2024 Christian Meeßen (GFZ) -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) // SPDX-FileCopyrightText: 2022 - 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.git; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - +import java.net.URI; import java.util.List; import java.util.Optional; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; class GithubScraperTest { @@ -23,11 +25,16 @@ class GithubScraperTest { @Test void givenListWithLastPageHeader_whenParsing_thenCorrectPageReturned() { - List singleLinkList = List.of("; rel=\"next\", ; rel=\"last\""); + List singleLinkList = List.of( + "; rel=\"next\", ; rel=\"last\"" + ); String[] lastPageData = GithubScraper.lastPageFromLinkHeader(singleLinkList); Assertions.assertEquals(2, lastPageData.length); - Assertions.assertEquals("https://api.github.com/repositories/413814951/contributors?per_page=1&page=9", lastPageData[0]); + Assertions.assertEquals( + "https://api.github.com/repositories/413814951/contributors?per_page=1&page=9", + lastPageData[0] + ); Assertions.assertEquals("9", lastPageData[1]); } @@ -67,4 +74,44 @@ void givenGitRepoUrl_whenCreatingScraper_thenRemoveGitSuffix() { Optional scraper = GithubScraper.create(githubUrlPrefix + repoGit); Assertions.assertEquals(repo, scraper.get().organisation + "/" + scraper.get().repo); } + + @Test + void givenFullIsArchivedResponse_whenParsing_thenCorrectResultReturned() { + String trueResponse = "{\"data\": {\"project\": {\"archived\": true}}}"; + Assertions.assertEquals(Boolean.TRUE, GitlabScraper.parseArchivedResponse(trueResponse)); + + String falseResponse = "{\"data\": {\"project\": {\"archived\": false}}}"; + Assertions.assertEquals(Boolean.FALSE, GitlabScraper.parseArchivedResponse(falseResponse)); + + String nullResponse = "{\"data\": {\"project\": {\"archived\": null}}}"; + Assertions.assertNull(GitlabScraper.parseArchivedResponse(nullResponse)); + } + + @ParameterizedTest + @ValueSource(strings = { "{\"data\": {\"project\": null}}", "{\"data\": null}", "{}" }) + void givenIncompleteIsArchivedResponse_whenParsing_thenCorrectResultReturned(String response) { + Assertions.assertNull(GitlabScraper.parseArchivedResponse(response)); + } + + @Test + void givenCorrectJson_whenParsingReadmeUrl_thenUrlReturned() { + String json = + "{\"download_url\": \"https://raw.githubusercontent.com/research-software-directory/RSD-as-a-service/main/README.md\"}"; + + Optional parsedUrl = GithubScraper.parseReadmeUrl(json); + + Assertions.assertTrue(parsedUrl.isPresent()); + Assertions.assertEquals( + URI.create("https://raw.githubusercontent.com/research-software-directory/RSD-as-a-service/main/README.md"), + parsedUrl.get() + ); + } + + @ParameterizedTest + @ValueSource(strings = { "{}", "{\"download_url\": null}", "{\"download_url\": 123}" }) + void givenMissingOrIncorrectlyStructuredJson_whenParsingReadmeUrl_thenEmptyValueReturned(String json) { + Optional parsedUrl = GithubScraper.parseReadmeUrl(json); + + Assertions.assertTrue(parsedUrl.isEmpty()); + } } diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GitlabScraperIT.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GitlabScraperIT.java index f3e4b5f..83d66fd 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GitlabScraperIT.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GitlabScraperIT.java @@ -11,8 +11,8 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; - class GitlabScraperIT { + private final String baseApiUrl = "https://git.gfz-potsdam.de/api"; private final String repo = "swc-bb/swc-templates/swc-l/python"; private final GitlabScraper scraper = new GitlabScraper(baseApiUrl, repo); diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GitlabScraperTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GitlabScraperTest.java new file mode 100644 index 0000000..c12fcf6 --- /dev/null +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/GitlabScraperTest.java @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2026 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +package nl.esciencecenter.rsd.scraper.git; + +import java.net.URI; +import java.util.Optional; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class GitlabScraperTest { + + @Test + void givenCorrectJson_whenParsingReadmeUrl_thenUrlReturned() { + String json = "{\"readme_url\": \"https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/README.md\"}"; + + Optional parsedUrl = GitlabScraper.parseReadmeUrl(json); + + Assertions.assertTrue(parsedUrl.isPresent()); + Assertions.assertEquals( + URI.create("https://gitlab.com/gitlab-org/gitlab-shell/-/raw/main/README.md"), + parsedUrl.get() + ); + } + + @ParameterizedTest + @ValueSource(strings = { "{}", "{\"readme_url\": null}", "{\"readme_url\": 123}" }) + void givenMissingOrIncorrectlyStructuredJson_whenParsingReadmeUrl_thenEmptyValueReturned(String json) { + Optional parsedUrl = GitlabScraper.parseReadmeUrl(json); + + Assertions.assertTrue(parsedUrl.isEmpty()); + } +} diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnectorTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnectorTest.java index 4a53d3c..de47423 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnectorTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/git/PostgrestConnectorTest.java @@ -1,15 +1,15 @@ -// SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2022 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 - 2026 Netherlands eScience Center +// SPDX-FileCopyrightText: 2025 Dusan Mijatovic (Netherlands eScience Center) // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.git; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - import java.util.Collection; import java.util.UUID; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; class PostgrestConnectorTest { @@ -25,12 +25,12 @@ void givenEmptyJsonArray_whenParsingJsonData_thenEmptyCollectionReturned() { @Test void givenValidJson_whenParsingJsonData_thenRecordWithDataReturned() { String arrayWithSingleValidObjectJson = """ - [ - { - "software": "3a07a021-743e-4adf-a2d9-3c85075fe9cf", - "url": "https://www.example.com" - } - ]"""; + [ + { + "id": "3a07a021-743e-4adf-a2d9-3c85075fe9cf", + "url": "https://www.example.com" + } + ]"""; Collection result = PostgrestConnector.parseBasicJsonData(arrayWithSingleValidObjectJson); @@ -43,26 +43,28 @@ void givenValidJson_whenParsingJsonData_thenRecordWithDataReturned() { @Test void givenJsonWithInvalidUuid_whenParsingJsonData_thenExceptionThrown() { String invalidUuidJson = """ - [ - { - "software": "not-a-UUID", - "url": "https://www.example.com" - } - ]"""; - - Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> PostgrestConnector.parseBasicJsonData(invalidUuidJson)); + [ + { + "id": "not-a-UUID", + "url": "https://www.example.com" + } + ]"""; + + Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> + PostgrestConnector.parseBasicJsonData(invalidUuidJson) + ); } @Test void givenInvalidJson_whenParsingJsonData_thenExceptionThrown() { -// missing comma after software UUID + // missing comma after id UUID String invalidJson = """ - [ - { - "software": "3a07a021-743e-4adf-a2d9-3c85075fe9cf" - "url": "https://www.example.com" - } - ]"""; + [ + { + "id": "3a07a021-743e-4adf-a2d9-3c85075fe9cf" + "url": "https://www.example.com" + } + ]"""; Assertions.assertThrows(RuntimeException.class, () -> PostgrestConnector.parseBasicJsonData(invalidJson)); } @@ -70,20 +72,22 @@ void givenInvalidJson_whenParsingJsonData_thenExceptionThrown() { @Test void givenValidJsonWithMissingFields_whenParsingJsonData_thenExceptionThrown() { String missingSoftwareJson = """ - [ - { - "url": "https://www.example.com" - } - ]"""; + [ + { + "url": "https://www.example.com" + } + ]"""; - Assertions.assertThrows(RuntimeException.class, () -> PostgrestConnector.parseBasicJsonData(missingSoftwareJson)); + Assertions.assertThrows(RuntimeException.class, () -> + PostgrestConnector.parseBasicJsonData(missingSoftwareJson) + ); String missingUrlJson = """ - [ - { - "software": "3a07a021-743e-4adf-a2d9-3c85075fe9cf" - } - ]"""; + [ + { + "id": "3a07a021-743e-4adf-a2d9-3c85075fe9cf" + } + ]"""; Assertions.assertThrows(RuntimeException.class, () -> PostgrestConnector.parseBasicJsonData(missingUrlJson)); } @@ -91,22 +95,22 @@ void givenValidJsonWithMissingFields_whenParsingJsonData_thenExceptionThrown() { @Test void givenValidJsonWithNullFields_whenParsingJsonData_thenExceptionThrown() { String nullSoftwareJson = """ - [ - { - "software": null, - "url": "https://www.example.com" - } - ]"""; + [ + { + "id": null, + "url": "https://www.example.com" + } + ]"""; Assertions.assertThrows(RuntimeException.class, () -> PostgrestConnector.parseBasicJsonData(nullSoftwareJson)); String nullUrlJson = """ - [ - { - "software": "3a07a021-743e-4adf-a2d9-3c85075fe9cf", - "url": null - } - ]"""; + [ + { + "id": "3a07a021-743e-4adf-a2d9-3c85075fe9cf", + "url": null + } + ]"""; Assertions.assertThrows(RuntimeException.class, () -> PostgrestConnector.parseBasicJsonData(nullUrlJson)); } diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepositoryTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepositoryTest.java index bc32ac4..67f99a8 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepositoryTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/license/GitHubSpdxLicenseRepositoryTest.java @@ -1,69 +1,70 @@ -// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.license; +import java.util.Map; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.Map; - class GitHubSpdxLicenseRepositoryTest { @Test void parseLicensesJson() { // editorconfig-checker-disable String json = """ + { + "licenseListVersion": "779bbe0", + "licenses": [ + { + "reference": "https://spdx.org/licenses/0BSD.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/0BSD.json", + "referenceNumber": 16, + "name": "BSD Zero Clause License", + "licenseId": "0BSD", + "seeAlso": [ + "http://landley.net/toybox/license.html", + "https://opensource.org/licenses/0BSD" + ], + "isOsiApproved": true + }, + { + "reference": "https://spdx.org/licenses/MIT.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/MIT.json", + "referenceNumber": 535, + "name": "MIT License", + "licenseId": "MIT", + "seeAlso": [ + "https://opensource.org/license/mit/" + ], + "isOsiApproved": true, + "isFsfLibre": true + }, { - "licenseListVersion": "779bbe0", - "licenses": [ - { - "reference": "https://spdx.org/licenses/0BSD.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/0BSD.json", - "referenceNumber": 16, - "name": "BSD Zero Clause License", - "licenseId": "0BSD", - "seeAlso": [ - "http://landley.net/toybox/license.html", - "https://opensource.org/licenses/0BSD" - ], - "isOsiApproved": true - }, - { - "reference": "https://spdx.org/licenses/MIT.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/MIT.json", - "referenceNumber": 535, - "name": "MIT License", - "licenseId": "MIT", - "seeAlso": [ - "https://opensource.org/license/mit/" - ], - "isOsiApproved": true, - "isFsfLibre": true - }, - { - "reference": "https://spdx.org/licenses/CC-BY-4.0.html", - "isDeprecatedLicenseId": false, - "detailsUrl": "https://spdx.org/licenses/CC-BY-4.0.json", - "referenceNumber": 578, - "name": "Creative Commons Attribution 4.0 International", - "licenseId": "CC-BY-4.0", - "seeAlso": [ - "https://creativecommons.org/licenses/by/4.0/legalcode" - ], - "isOsiApproved": false, - "isFsfLibre": true - } + "reference": "https://spdx.org/licenses/CC-BY-4.0.html", + "isDeprecatedLicenseId": false, + "detailsUrl": "https://spdx.org/licenses/CC-BY-4.0.json", + "referenceNumber": 578, + "name": "Creative Commons Attribution 4.0 International", + "licenseId": "CC-BY-4.0", + "seeAlso": [ + "https://creativecommons.org/licenses/by/4.0/legalcode" ], - "releaseDate": "2024-10-10" - }"""; + "isOsiApproved": false, + "isFsfLibre": true + } + ], + "releaseDate": "2024-10-10" + }"""; // editorconfig-checker-enable - Map licenseMap = Assertions.assertDoesNotThrow(() -> GitHubSpdxLicenseRepository.parseLicensesJson(json)); + Map licenseMap = Assertions.assertDoesNotThrow(() -> + GitHubSpdxLicenseRepository.parseLicensesJson(json) + ); Assertions.assertEquals(3, licenseMap.size()); Assertions.assertTrue(licenseMap.containsKey("0BSD")); Assertions.assertTrue(licenseMap.containsKey("MIT")); diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntryTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntryTest.java index 1f4c7ac..df2e8f8 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntryTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/nassa/NassaSoftwareEntryTest.java @@ -1,60 +1,63 @@ -// SPDX-FileCopyrightText: 2024 - 2025 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2024 - 2025 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 package nl.esciencecenter.rsd.scraper.nassa; +import java.io.StringReader; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.io.StringReader; - class NassaSoftwareEntryTest { @Test void givenValidNassaYaml_whenParsing_thenSoftwareEntryReturned() { // editorconfig-checker-disable String validYaml = """ - id: 2022-Verhagen-001 - nassaVersion: 0.5.0 - moduleType: Algorithm - title: Determine fertility rates for use in a demographic simulation. - moduleVersion: 1.0.1 - contributors: - - name: Verhagen, Philip - roles: [ "Author", "Copyright Holder", "Creator" ] - email: j.w.h.p.verhagen@vu.nl - orcid: 0000-0001-8166-122X - lastUpdateDate: 2022-12-02 - description: > - This NetLogo code snippet provides fertility estimates of females for use in demographic simulations. The figures used are based on two different sources, and will be assigned per 5-year age cohort. - relatedModules:\s - references: - moduleReferences: [ Bagnall-Frier-1994, Coale-Trussell-1974, Coale-Trussell-1978, Henry-1961, Verhagen-2022 ] - domainKeywords: - regions:\s - periods:\s - - Roman period - - Modern period - subjects:\s - - demography - - fertility - modellingKeywords: - - parameter initialisation - programmingKeywords: - - Functional - - Object-Oriented - implementations: - - language: NetLogo - softwareDependencies:\s - - NetLogo 6.2.0 - docsDir: documentation/ - license: MIT - """; + id: 2022-Verhagen-001 + nassaVersion: 0.5.0 + moduleType: Algorithm + title: Determine fertility rates for use in a demographic simulation. + moduleVersion: 1.0.1 + contributors: + - name: Verhagen, Philip + roles: [ "Author", "Copyright Holder", "Creator" ] + email: j.w.h.p.verhagen@vu.nl + orcid: 0000-0001-8166-122X + lastUpdateDate: 2022-12-02 + description: > + This NetLogo code snippet provides fertility estimates of females for use in demographic simulations. The figures used are based on two different sources, and will be assigned per 5-year age cohort. + relatedModules:\s + references: + moduleReferences: [ Bagnall-Frier-1994, Coale-Trussell-1974, Coale-Trussell-1978, Henry-1961, Verhagen-2022 ] + domainKeywords: + regions:\s + periods:\s + - Roman period + - Modern period + subjects:\s + - demography + - fertility + modellingKeywords: + - parameter initialisation + programmingKeywords: + - Functional + - Object-Oriented + implementations: + - language: NetLogo + softwareDependencies:\s + - NetLogo 6.2.0 + docsDir: documentation/ + license: MIT + """; // editorconfig-checker-enable - NassaSoftwareEntry entry = NassaSoftwareEntry.fromYaml(new StringReader(validYaml), "https://github.com/Archaeology-ABM/NASSA-modules/tree/main/2022-Verhagen-001", null); + NassaSoftwareEntry entry = NassaSoftwareEntry.fromYaml( + new StringReader(validYaml), + "https://github.com/Archaeology-ABM/NASSA-modules/tree/main/2022-Verhagen-001", + null + ); Assertions.assertNotNull(entry); Assertions.assertEquals("Determine fertility rates for use in a demographic simulation.", entry.title); diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraperTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraperTest.java index 2c527c3..0f4e1dd 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraperTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CranScraperTest.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2023 - 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2023 - 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2023 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2023 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 @@ -15,21 +15,25 @@ class CranScraperTest { @ParameterizedTest @CsvSource({ - "https://CRAN.R-project.org/package=TIMP,TIMP", - "https://CRAN.R-project.org/package=TIMP/,TIMP", - "https://cran.r-project.org/package=splithalfr,splithalfr", - "https://cran.r-project.org/web/packages/GGIR,GGIR", - "https://cran.r-project.org/web/packages/GGIR/,GGIR", - "https://cran.r-project.org/web/packages/GGIR/index.html,GGIR", - "https://cran.r-project.org/web/packages/GGIR/index.html/,GGIR" + "https://CRAN.R-project.org/package=TIMP,TIMP", + "https://CRAN.R-project.org/package=TIMP/,TIMP", + "https://cran.r-project.org/package=splithalfr,splithalfr", + "https://cran.r-project.org/web/packages/GGIR,GGIR", + "https://cran.r-project.org/web/packages/GGIR/,GGIR", + "https://cran.r-project.org/web/packages/GGIR/index.html,GGIR", + "https://cran.r-project.org/web/packages/GGIR/index.html/,GGIR", }) - void givenValidCranUrl_whenCallingConstructor_thenNoExceptionThrownAndPackageNamesCorrect(String url, String expectedPackageName) { + void givenValidCranUrl_whenCallingConstructor_thenNoExceptionThrownAndPackageNamesCorrect( + String url, + String expectedPackageName + ) { CranScraper cranScraper = Assertions.assertDoesNotThrow(() -> new CranScraper(url)); Assertions.assertEquals(expectedPackageName, cranScraper.packageName); } @ParameterizedTest - @ValueSource(strings = { + @ValueSource( + strings = { "https://cran.r-project.org/web/packages/", "https://cran.r-project.org/web/packages/GG IR", "https://cran.r-project.org/web/packages/GGIR ", @@ -39,8 +43,9 @@ void givenValidCranUrl_whenCallingConstructor_thenNoExceptionThrownAndPackageNam "https://CRAN.R-project.org/package=TI MP ", "https://CRAN.R-project.org/package=TIMP /", "https://www.example.com", - "" - }) + "", + } + ) void givenInvalidCranUrl_whenCallingConstructor_thenExceptionThrown(String url) { Assertions.assertThrowsExactly(RuntimeException.class, () -> new CranScraper(url)); } diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraperTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraperTest.java index b0dacb0..3d9185b 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraperTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/CratesScraperTest.java @@ -13,10 +13,7 @@ class CratesScraperTest { @ParameterizedTest - @CsvSource({ - "https://crates.io/crates/tokio,tokio", - "https://crates.io/crates/tokio/,tokio", - }) + @CsvSource({ "https://crates.io/crates/tokio,tokio", "https://crates.io/crates/tokio/,tokio" }) void givenValidCratesUrl_whenCallingConstructor_thenNoExceptionThrownAndPackageNamesCorrect( String url, String expectedPackageName diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraperTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraperTest.java index 452c671..6ab201f 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraperTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/GoScraperTest.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 diff --git a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraperTest.java b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraperTest.java index 2f7cf99..d7bebb5 100644 --- a/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraperTest.java +++ b/scrapers/src/test/java/nl/esciencecenter/rsd/scraper/package_manager/scrapers/MavenScraperTest.java @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) -// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// SPDX-FileCopyrightText: 2024 - 2026 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 - 2026 Netherlands eScience Center // // SPDX-License-Identifier: Apache-2.0 @@ -31,15 +31,17 @@ void givenValidMavenOrSonatypeUrl_whenCallingConstructor_thenNoExceptionThrownAn } @ParameterizedTest - @ValueSource(strings = { - "https://central.sonatype.com/artifact", - "https://mvnrepository.com/artifact/", - "https://central.sonatype.com/artifact/org.openscience.cdk", - "https://mvnrepository.com/artifact/io.github.sanctuuary/", - "https://www.example.com", - "https://www.example.com/artifact/org.openscience.cdk/cdk-bundle", - "" - }) + @ValueSource( + strings = { + "https://central.sonatype.com/artifact", + "https://mvnrepository.com/artifact/", + "https://central.sonatype.com/artifact/org.openscience.cdk", + "https://mvnrepository.com/artifact/io.github.sanctuuary/", + "https://www.example.com", + "https://www.example.com/artifact/org.openscience.cdk/cdk-bundle", + "", + } + ) void givenInvalidUrl_whenCallingConstructor_thenExceptionThrown(String url) { Assertions.assertThrowsExactly(RuntimeException.class, () -> new MavenScraper(url)); }