Skip to content

fix(security): block SSRF in OAI-ORE harvest ingestion (GHSA-qff6) #1561

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Kasinhou wants to merge 3 commits into
base: dtq-dev
Choose a base branch
from
Open

fix(security): block SSRF in OAI-ORE harvest ingestion (GHSA-qff6) #1561

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f8ff37e
Harden ORE harvest ingest against SSRF
Oct 27, 2026
e6b3d32
Add DNS-pinning test and redirect note for ORE egress guard
Oct 27, 2026
6eb0633
Address review: redact rejected URLs, close fetched stream, pre-valid…
Oct 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
182 changes: 82 additions & 100 deletions dspace-api/src/main/java/org/dspace/content/crosswalk/OREIngestionCrosswalk.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,17 @@
*/
package org.dspace.content.crosswalk;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.ConnectException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.sql.SQLException;
import java.text.NumberFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Set;

import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.logging.log4j.Logger;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.Bitstream;
Expand All @@ -38,6 +32,12 @@
import org.dspace.content.service.ItemService;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.harvest.ore.HarvestPolicyAware;
import org.dspace.harvest.ore.OreEgressPolicy;
import org.dspace.harvest.ore.OreResourceRejectedException;
import org.dspace.harvest.ore.OreUrlValidator;
import org.dspace.harvest.ore.RejectionReason;
import org.dspace.harvest.ore.SafeResourceFetcher;
import org.dspace.services.ConfigurationService;
import org.dspace.services.factory.DSpaceServicesFactory;
import org.jdom2.Attribute;
Expand All @@ -56,7 +56,7 @@
* @author Alexey Maslov
*/
public class OREIngestionCrosswalk
implements IngestionCrosswalk {
implements IngestionCrosswalk, HarvestPolicyAware {
/**
* log4j category
*/
Expand Down Expand Up @@ -84,6 +84,25 @@ public class OREIngestionCrosswalk
protected ItemService itemService = ContentServiceFactory.getInstance().getItemService();
protected ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService();

private final SafeResourceFetcher resourceFetcher = new SafeResourceFetcher();
private OreEgressPolicy oreEgressPolicy;

@Override
public void setOreEgressPolicy(OreEgressPolicy policy) {
this.oreEgressPolicy = policy;
}

/**
* The packagers and the XSLT CLI reach this crosswalk with no harvest context, so they get the
* fail-closed policy rather than no policy at all.
*/
private OreEgressPolicy egressPolicy() {
if (oreEgressPolicy == null) {
oreEgressPolicy = OreEgressPolicy.strictest(configurationService);
}
return oreEgressPolicy;
}


@Override
public void ingest(Context context, DSpaceObject dso, List<Element> metadata, boolean createMissingMetadataFields)
Expand Down Expand Up @@ -135,6 +154,23 @@ public void ingest(Context context, DSpaceObject dso, Element root, boolean crea
Filters.attribute(), null, ATOM_NS);
entryId = xpathAltHref.evaluateFirst(doc).getValue();

// Validate every aggregated resource URL up front, before any bitstream is created, so a URL rejected
// by the egress policy cannot leave the bitstreams of earlier resources orphaned in the assetstore
// when the record is rolled back. The fetch below still re-validates each URL (and every redirect hop)
// to guard against TOCTOU.
OreUrlValidator preValidator = new OreUrlValidator();
for (Element resource : aggregatedResources) {
String href = resource.getAttributeValue("href");
if (href == null) {
throw new CrosswalkException("Entry did not contain link to resource: " + entryId);
}
OreUrlValidator.Decision decision =
preValidator.validate(OreUrlValidator.parse(encodeForURL(href)), egressPolicy());
if (!decision.isAllowed()) {
throw new OreResourceRejectedException(decision.getReason(), href, decision.getDetail());
}
}

// Next for each resource, create a bitstream
NumberFormat nf = NumberFormat.getInstance();
nf.setGroupingUsed(false);
Expand Down Expand Up @@ -174,49 +210,36 @@ public void ingest(Context context, DSpaceObject dso, Element root, boolean crea
targetBundle = targetBundles.get(0);
}

URL ARurl = null;
InputStream in = null;
if (href != null) {
try {
// Make sure the url string escapes all the oddball characters
String processedURL = encodeForURL(href);
if (validResourceUri(entryId, processedURL)) {
// Generate a request for the aggregated resource
ARurl = new URL(processedURL);
in = ARurl.openStream();
} else {
throw new FileNotFoundException("Failed to validate " + processedURL);
}
} catch (FileNotFoundException fe) {
log.error("The provided URI failed to return a resource: " + href);
} catch (ConnectException fe) {
log.error("The provided URI was invalid: " + href);
}
} else {
throw new CrosswalkException("Entry did not contain link to resource: " + entryId);
// Make sure the url string escapes all the oddball characters
String processedURL = encodeForURL(href);
Bitstream newBitstream;
// try-with-resources so the fetched stream - and the underlying HTTP connection/client it holds -
// is always closed, including when bitstream creation fails part way through.
// The trust anchor of the egress policy is the collection's oai_source, not entryId: entryId is
// read from this remote document and is therefore attacker-controlled.
try (InputStream in = resourceFetcher.fetch(OreUrlValidator.parse(processedURL), egressPolicy())) {
newBitstream = bitstreamService.create(context, targetBundle, in);
} catch (IOException ioe) {
// a transport failure - or the size cap, which the bitstore surfaces here wrapped in an
// IOException - has to drop this record only; escaping would stop the whole collection and,
// after removeAllBundles, commit the item without its files
throw transferFailed(href, ioe);
}

// ingest and update
if (in != null) {
Bitstream newBitstream = bitstreamService.create(context, targetBundle, in);

String bsName = resource.getAttributeValue("title");
newBitstream.setName(context, bsName);
String bsName = resource.getAttributeValue("title");
newBitstream.setName(context, bsName);

// Identify the format
String mimeString = resource.getAttributeValue("type");
BitstreamFormat bsFormat = bitstreamFormatService.findByMIMEType(context, mimeString);
if (bsFormat == null) {
bsFormat = bitstreamFormatService.guessFormat(context, newBitstream);
}
newBitstream.setFormat(context, bsFormat);
bitstreamService.update(context, newBitstream);

bundleService.addBitstream(context, targetBundle, newBitstream);
bundleService.update(context, targetBundle);
} else {
throw new CrosswalkException("Could not retrieve bitstream: " + entryId);
// Identify the format
String mimeString = resource.getAttributeValue("type");
BitstreamFormat bsFormat = bitstreamFormatService.findByMIMEType(context, mimeString);
if (bsFormat == null) {
bsFormat = bitstreamFormatService.guessFormat(context, newBitstream);
}
newBitstream.setFormat(context, bsFormat);
bitstreamService.update(context, newBitstream);

bundleService.addBitstream(context, targetBundle, newBitstream);
bundleService.update(context, targetBundle);

}
log.info(
Expand Down Expand Up @@ -263,60 +286,19 @@ private String encodeForURL(String sourceString) {
}

/**
* Validate a resource URI against the host and scheme of the remote OAI endpoint, or a configured
* list of allowed prefixes.
* This still implicitly "trusts" the remote OAI server, but will reject resource URIs with a totally
* different hostname to avoid downloading malicious resources from a compromised endpoint.
* Even if the URL prefix validation is disabled, schemes will still be enforced to http(s) so file:/// and
* other unwanted schemes cannot be used
* @param entryUrl the entryId of the parent ORE resource
* @param resourceUrl the resource URL of the aggregated ORE resource
* @return result of the validation
* Turn an I/O failure while fetching or storing a file into a rejection of this one record.
*
* @param href the resource that could not be transferred
* @param cause the failure, kept so the real reason still reaches the log
* @return the rejection to throw
*/
private boolean validResourceUri(String entryUrl, String resourceUrl) {
try {
Set<String> allowedSchemes = Set.of("http", "https");
URI entryUri = new URI(entryUrl).normalize();
URI resourceUri = new URI(resourceUrl).normalize();
String scheme = resourceUri.getScheme();

if (scheme == null ||
!allowedSchemes.contains(scheme.toLowerCase(Locale.ROOT))) {
log.warn("Illegal scheme requested for ORE resource: {}", resourceUri);
return false;
}

if (configurationService.getBooleanProperty("oai.harvester.ore.file.validateUrlPrefix", false)) {
for (String allowedPrefix : configurationService
.getArrayProperty("oai.harvester.ore.file.allowedUrlPrefix")) {
URI allowedUri = new URI(allowedPrefix).normalize();
// Return true on the first allowed prefix match
if (Objects.equals(resourceUri.getScheme(), allowedUri.getScheme())
&& Objects.equals(resourceUri.getHost().toLowerCase(Locale.ROOT),
allowedUri.getHost().toLowerCase(Locale.ROOT))) {
return true;
}
}

// If no allowed prefixes were matched, we require scheme + host to match the remote OAI server
if (!Objects.equals(entryUri.getScheme(), resourceUri.getScheme())) {
log.warn("Illegal scheme requested for ORE resource: {}", resourceUri);
return false;
}
if (!Objects.equals(
entryUri.getHost().toLowerCase(Locale.ROOT),
resourceUri.getHost().toLowerCase(Locale.ROOT))) {
log.warn("Illegal host requested for ORE resource: {}", resourceUri);
return false;
}
}

return true;

} catch (URISyntaxException e) {
log.warn("Could not validate ORE resource URI: {}", resourceUrl);
return false;
}
private OreResourceRejectedException transferFailed(String href, IOException cause) {
// the cap is enforced while the body streams, so it arrives wrapped rather than as itself
boolean tooLarge =
ExceptionUtils.indexOfType(cause, SafeResourceFetcher.ResponseTooLargeException.class) >= 0;
return new OreResourceRejectedException(
tooLarge ? RejectionReason.RESPONSE_TOO_LARGE : RejectionReason.FETCH_FAILED, href,
tooLarge ? "size cap exceeded" : "transfer failed", cause);
}

}
Loading
Loading