Skip to content

Bump dependencies and upgrade to Tika 4.1.0 #8141

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
labkey-adam merged 9 commits into from
Oct 8, 2026
Merged

Bump dependencies and upgrade to Tika 4.1.0 #8141

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
0f113d1
Bump dependencies
labkey-adam Oct 6, 2026
b6e8fb6
Deprecate PostgreSQL 14.x
labkey-adam Oct 6, 2026
8741c42
Upgrade Tika 3.3.2 -> 4.1.0
labkey-adam Oct 7, 2026
7ca97b7
Add parsing support and testing for eml and sas7bdat files
labkey-adam Oct 7, 2026
46b9eed
Extract properties for all remaining hard-coded versions
labkey-adam Oct 7, 2026
e4f0837
Exclude parsers we don't want to prevent warning logging
labkey-adam Oct 7, 2026
2112c8e
Update comments
labkey-adam Oct 7, 2026
f0e3d85
Share the DefaultDetector
labkey-adam Oct 7, 2026
da5b1a5
Update comments
labkey-adam Oct 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion api/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1019,7 +1019,11 @@ dependencies {
ExternalDependency.APACHE_2_LICENSE_NAME,
ExternalDependency.APACHE_2_LICENSE_URL,
"Just detection of file content types and extraction of text (no parsers)",
)
),
{
// Only Tika's Markdown content handler needs commonmark; core module ships its own copy
exclude group: "org.commonmark"
}
)

BuildUtils.addExternalDependency(
Expand Down
6 changes: 4 additions & 2 deletions api/src/org/labkey/api/util/FileType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,10 @@
import org.apache.tika.detect.Detector;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.mime.MimeTypes;
import org.apache.tika.parser.ParseContext;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.junit.Assert;
Expand Down Expand Up @@ -586,10 +588,10 @@ public boolean isType(@Nullable String filePath, @Nullable String contentType, b
protected static String detectContentType(String fileName, byte[] header)
{
final Metadata metadata = new Metadata();
metadata.set("resourceName", fileName);
metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, fileName);
try (TikaInputStream is = TikaInputStream.get(header, metadata))
{
MediaType mediaType = DETECTOR.detect(is, metadata);
MediaType mediaType = DETECTOR.detect(is, metadata, new ParseContext());
if (mediaType != null)
return mediaType.toString();

Expand Down
22 changes: 13 additions & 9 deletions api/src/org/labkey/api/util/PageFlowUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,16 @@
import org.apache.commons.lang3.Strings;
import org.apache.logging.log4j.Logger;
import org.apache.tika.detect.DefaultDetector;
import org.apache.tika.detect.Detector;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.mime.MimeType;
import org.apache.tika.mime.MimeTypeException;
import org.apache.tika.mime.MimeTypes;
import org.apache.tika.parser.ParseContext;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.json.JSONObject;
Expand Down Expand Up @@ -928,19 +931,20 @@ public static MediaType getMediaTypeFor(File file)
return getMediaTypeFor(file.toPath());
}

private static final Detector DETECTOR = new DefaultDetector();

public static MediaType getMediaTypeFor(Path file)
{
try
{
DefaultDetector detector = new DefaultDetector();
Metadata metaData = new Metadata();
String filename = file.getFileName().toString();
Metadata metaData = new Metadata();
String filename = file.getFileName().toString();

// use the metadata to hint at the type for a faster lookup
metaData.add(TikaCoreProperties.RESOURCE_NAME_KEY, filename);
metaData.add(Metadata.CONTENT_TYPE, PageFlowUtil.getContentTypeFor(filename));
// use the metadata to hint at the type for a faster lookup
metaData.add(TikaCoreProperties.RESOURCE_NAME_KEY, filename);
metaData.add(HttpHeaders.CONTENT_TYPE, PageFlowUtil.getContentTypeFor(filename));

return detector.detect(TikaInputStream.get(file), metaData);
try (TikaInputStream tis = TikaInputStream.get(file))
{
return DETECTOR.detect(tis, metaData, new ParseContext());
}
catch (IOException e)
{
Expand Down
61 changes: 31 additions & 30 deletions core/src/org/labkey/core/dialect/PostgreSqlVersion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,13 @@
public enum PostgreSqlVersion
{
POSTGRESQL_UNSUPPORTED(-1, true, false, null),
POSTGRESQL_14(140, false, true, PostgreSql_14_Dialect::new),
POSTGRESQL_14(140, true, true, PostgreSql_14_Dialect::new),
POSTGRESQL_15(150, false, true, PostgreSql_15_Dialect::new),
POSTGRESQL_16(160, false, true, PostgreSql_16_Dialect::new),
POSTGRESQL_17(170, false, true, PostgreSql_17_Dialect::new),
POSTGRESQL_18(180, false, true, PostgreSql_18_Dialect::new),
POSTGRESQL_19(190, false, false, PostgreSql_19_Dialect::new),
POSTGRESQL_FUTURE(Integer.MAX_VALUE, true, false, PostgreSql_19_Dialect::new);
POSTGRESQL_FUTURE(Integer.MAX_VALUE, false, false, PostgreSql_19_Dialect::new);

public static final String RECOMMENDED = PRODUCT_NAME + " 18.x is the recommended version.";

Expand Down Expand Up @@ -102,41 +102,42 @@ public static class TestCase extends Assert
public void test()
{
// Good
test(140, POSTGRESQL_14);
test(150, POSTGRESQL_15);
test(160, POSTGRESQL_16);
test(170, POSTGRESQL_17);
test(180, POSTGRESQL_18);
test(190, POSTGRESQL_19);
test(140, POSTGRESQL_14, true);
test(150, POSTGRESQL_15, false);
test(160, POSTGRESQL_16, false);
test(170, POSTGRESQL_17, false);
test(180, POSTGRESQL_18, false);
test(190, POSTGRESQL_19, false);

// Future
test(200, POSTGRESQL_FUTURE);
test(210, POSTGRESQL_FUTURE);
test(220, POSTGRESQL_FUTURE);
test(200, POSTGRESQL_FUTURE, false);
test(210, POSTGRESQL_FUTURE, false);
test(220, POSTGRESQL_FUTURE, false);

// Bad
test(83, POSTGRESQL_UNSUPPORTED);
test(84, POSTGRESQL_UNSUPPORTED);
test(85, POSTGRESQL_UNSUPPORTED);
test(90, POSTGRESQL_UNSUPPORTED);
test(91, POSTGRESQL_UNSUPPORTED);
test(92, POSTGRESQL_UNSUPPORTED);
test(93, POSTGRESQL_UNSUPPORTED);
test(94, POSTGRESQL_UNSUPPORTED);
test(95, POSTGRESQL_UNSUPPORTED);
test(96, POSTGRESQL_UNSUPPORTED);
test(97, POSTGRESQL_UNSUPPORTED);
test(98, POSTGRESQL_UNSUPPORTED);
test(99, POSTGRESQL_UNSUPPORTED);
test(100, POSTGRESQL_UNSUPPORTED);
test(110, POSTGRESQL_UNSUPPORTED);
test(120, POSTGRESQL_UNSUPPORTED);
test(130, POSTGRESQL_UNSUPPORTED);
test(83, POSTGRESQL_UNSUPPORTED, true);
test(84, POSTGRESQL_UNSUPPORTED, true);
test(85, POSTGRESQL_UNSUPPORTED, true);
test(90, POSTGRESQL_UNSUPPORTED, true);
test(91, POSTGRESQL_UNSUPPORTED, true);
test(92, POSTGRESQL_UNSUPPORTED, true);
test(93, POSTGRESQL_UNSUPPORTED, true);
test(94, POSTGRESQL_UNSUPPORTED, true);
test(95, POSTGRESQL_UNSUPPORTED, true);
test(96, POSTGRESQL_UNSUPPORTED, true);
test(97, POSTGRESQL_UNSUPPORTED, true);
test(98, POSTGRESQL_UNSUPPORTED, true);
test(99, POSTGRESQL_UNSUPPORTED, true);
test(100, POSTGRESQL_UNSUPPORTED, true);
test(110, POSTGRESQL_UNSUPPORTED, true);
test(120, POSTGRESQL_UNSUPPORTED, true);
test(130, POSTGRESQL_UNSUPPORTED, true);
}

private void test(int version, PostgreSqlVersion expectedVersion)
private void test(int version, PostgreSqlVersion expectedVersion, boolean expectedDeprecated)
{
Assert.assertEquals(get(version), expectedVersion);
Assert.assertEquals(expectedVersion, get(version));
Assert.assertEquals(expectedDeprecated, get(version).isDeprecated());
}
}
}
133 changes: 118 additions & 15 deletions search/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,13 @@ plugins {
id 'org.labkey.build.module'
}

ext.mime4jVersion="0.8.14"
// Sync with versions in Tika's parent POM when upgrading tikaVersion
ext.commonsCsvVersion="1.14.1"
ext.ddPlistVersion="1.30"
ext.jhighlightVersion="2.0.0"
ext.metadataExtractorVersion="2.21.0"
ext.mime4jVersion="0.8.15"
ext.parsoVersion="2.0.14"

dependencies {
BuildUtils.addExternalDependency(
Expand Down Expand Up @@ -272,9 +278,106 @@ dependencies {
}
)

BuildUtils.addExternalDependency(
project,
new ExternalDependency(
"org.apache.tika:tika-parser-datauri-commons:${tikaVersion}",
"Tika",
"Apache",
"http://lucene.apache.org/tika/",
ExternalDependency.APACHE_2_LICENSE_NAME,
ExternalDependency.APACHE_2_LICENSE_URL,
"Data URI support for HTML and text parsers"
),
{
transitive = false
}
)

// TODO Tika 4.2.0: no Tika module uses mail-commons anymore (https://issues.apache.org/jira/browse/TIKA-4917), so remove this dependency if that holds in the release
BuildUtils.addExternalDependency(
project,
new ExternalDependency(
"org.apache.tika:tika-parser-mail-commons:${tikaVersion}",
"Tika",
"Apache",
"http://lucene.apache.org/tika/",
ExternalDependency.APACHE_2_LICENSE_NAME,
ExternalDependency.APACHE_2_LICENSE_URL,
"Email utilities used by the .eml and .mbox parsers"
),
{
transitive = false
}
)

BuildUtils.addExternalDependency(
project,
new ExternalDependency(
"org.apache.tika:tika-parser-mail-module:${tikaVersion}",
"Tika",
"Apache",
"http://lucene.apache.org/tika/",
ExternalDependency.APACHE_2_LICENSE_NAME,
ExternalDependency.APACHE_2_LICENSE_URL,
"Parser adapters for .eml and .mbox email formats"

@labkey-jeckels labkey-jeckels Oct 8, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there demand for this?

@labkey-adam labkey-adam Oct 8, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I haven't heard huge demand for either new format. But they were easy and cheap to add. Some clients use SAS and all clients use email, so 🤷 .

@labkey-adam labkey-adam Oct 8, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might be interesting to add metrics for file types we don't index because we don't recognize them.

),
{
transitive = false
}
)

BuildUtils.addExternalDependency(
project,
new ExternalDependency(
"org.apache.tika:tika-encoding-detector-html:${tikaVersion}",
"Tika",
"Apache",
"http://lucene.apache.org/tika/",
ExternalDependency.APACHE_2_LICENSE_NAME,
ExternalDependency.APACHE_2_LICENSE_URL,
"Character encoding detection from HTML meta tags"
),
{
transitive = false
}
)

BuildUtils.addExternalDependency(
project,
new ExternalDependency(
"org.apache.tika:tika-encoding-detector-mojibuster:${tikaVersion}",
"Tika",
"Apache",
"http://lucene.apache.org/tika/",
ExternalDependency.APACHE_2_LICENSE_NAME,
ExternalDependency.APACHE_2_LICENSE_URL,
"Statistical character encoding detection for text files"
),
{
transitive = false
}
)

BuildUtils.addExternalDependency(
project,
new ExternalDependency(
"org.apache.tika:tika-ml-core:${tikaVersion}",
"Tika",
"Apache",
"http://lucene.apache.org/tika/",
ExternalDependency.APACHE_2_LICENSE_NAME,
ExternalDependency.APACHE_2_LICENSE_URL,
"Model support for Tika's statistical encoding detector"
),
{
transitive = false
}
)

// Additional parsing libraries (beyond POI, PDFBox, etc., which are already included in API) that we
// care about. Adjust these dependencies and versions whenever we upgrade Tika. Or if we want to add or remove a parser.
// See also tikaConfig.xml to exclude specific parsers or detectors.
// See also the LuceneSearchServiceImpl constructor to exclude specific parsers or detectors.
BuildUtils.addExternalDependency(
project,
new ExternalDependency(
Expand Down Expand Up @@ -304,26 +407,26 @@ dependencies {
BuildUtils.addExternalDependency(
project,
new ExternalDependency(
"org.apache.pdfbox:jempbox:1.8.17",
"jempbox",
"Apache",
"https://pdfbox.apache.org/2.0/dependencies.html",
"com.drewnoakes:metadata-extractor:${metadataExtractorVersion}",
"Metadata Extractor",
"drewnoakes",
"https://github.com/drewnoakes/metadata-extractor",
ExternalDependency.APACHE_2_LICENSE_NAME,
ExternalDependency.APACHE_2_LICENSE_URL,
"PDFBox dependency"
"Parsers that read metadata from image files"
)
)

BuildUtils.addExternalDependency(
project,
new ExternalDependency(
"com.drewnoakes:metadata-extractor:2.21.0",
"Metadata Extractor",
"drewnoakes",
"https://github.com/drewnoakes/metadata-extractor",
"com.epam:parso:${parsoVersion}",
"Parso",
"EPAM",
"https://github.com/epam/parso",
ExternalDependency.APACHE_2_LICENSE_NAME,
ExternalDependency.APACHE_2_LICENSE_URL,
"Parsers that read metadata from image files"
"Parser for SAS .sas7bdat datasets"

@labkey-jeckels labkey-jeckels Oct 8, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there demand for this?

)
)

Expand All @@ -343,7 +446,7 @@ dependencies {
BuildUtils.addExternalDependency(
project,
new ExternalDependency(
"org.codelibs:jhighlight:2.0.0",
"org.codelibs:jhighlight:${jhighlightVersion}",
"JHighlight",
"codelibs",
"https://github.com/codelibs/jhighlight",
Expand All @@ -356,7 +459,7 @@ dependencies {
BuildUtils.addExternalDependency(
project,
new ExternalDependency(
"org.apache.commons:commons-csv:1.14.1",
"org.apache.commons:commons-csv:${commonsCsvVersion}",
"Commons CSV ™",
"Apache",
"http://commons.apache.org/proper/commons-csv/",
Expand Down Expand Up @@ -421,7 +524,7 @@ dependencies {
BuildUtils.addExternalDependency(
project,
new ExternalDependency(
"com.googlecode.plist:dd-plist:1.30",
"com.googlecode.plist:dd-plist:${ddPlistVersion}",
"DD Plist",
"com.dd.plist",
"https://github.com/3breadt/dd-plist",
Expand Down
Loading
Loading