Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 25 additions & 18 deletions be/src/format_v2/json/json_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -359,7 +359,6 @@ Status JsonReader::open(std::shared_ptr<FileScanRequest> request) {
}
RETURN_IF_ERROR(_parse_jsonpath_and_json_root());
_json_parser = std::make_unique<simdjson::ondemand::parser>();
_padding_buffer.resize(_padded_size);
_reader_eof = false;
_single_document_read = false;
_eof = false;
Expand Down Expand Up @@ -653,11 +652,11 @@ Status JsonReader::_read_one_document(size_t* size, bool* eof) {
*eof = true;
return Status::OK();
}
_document_buffer.resize(cast_set<size_t>(read_size));
Slice result(_document_buffer.data(), _document_buffer.size());
const auto document_size = cast_set<size_t>(read_size);
_grow_padding_buffer(document_size);
Slice result(_padding_buffer.data(), document_size);
RETURN_IF_ERROR(_physical_file_reader->read_at(_current_offset, result, size, _io_ctx.get()));
_document_buffer.resize(*size);
_document_view = _document_buffer;
_document_view = std::string_view(_padding_buffer.data(), *size);
if (*size == 0) {
*eof = true;
}
Expand All @@ -672,12 +671,12 @@ Status JsonReader::_read_one_document_from_pipe(size_t* read_size) {
DorisUniqueBufferPtr<uint8_t> file_buf;
RETURN_IF_ERROR(stream_load_pipe->read_one_message(&file_buf, read_size));
if (*read_size == 0) {
_document_buffer.clear();
_document_view = {};
return Status::OK();
}
_document_buffer.assign(reinterpret_cast<const char*>(file_buf.get()), *read_size);
_document_view = _document_buffer;
_grow_padding_buffer(*read_size);
std::memcpy(_padding_buffer.data(), file_buf.get(), *read_size);
_document_view = std::string_view(_padding_buffer.data(), *read_size);
if (!stream_load_pipe->is_chunked_transfer()) {
return Status::OK();
}
Expand All @@ -689,13 +688,21 @@ Status JsonReader::_read_one_document_from_pipe(size_t* read_size) {
if (next_size == 0) {
break;
}
_document_buffer.append(reinterpret_cast<const char*>(next_buf.get()), next_size);
_grow_padding_buffer(*read_size + next_size);
std::memcpy(_padding_buffer.data() + *read_size, next_buf.get(), next_size);
*read_size += next_size;
}
_document_view = _document_buffer;
_document_view = std::string_view(_padding_buffer.data(), *read_size);
return Status::OK();
}

void JsonReader::_grow_padding_buffer(size_t size) {
const size_t needed = size + simdjson::SIMDJSON_PADDING;
if (_padding_buffer.size() < needed) {
_padding_buffer.resize(needed);
}
}

Status JsonReader::_parse_next_json(size_t* size, bool* eof) {
RETURN_IF_ERROR(_read_one_document(size, eof));
if (*eof || *size == 0) {
Expand All @@ -707,17 +714,17 @@ Status JsonReader::_parse_next_json(size_t* size, bool* eof) {
_document_view.remove_prefix(3);
*size -= 3;
}
if (*size + simdjson::SIMDJSON_PADDING > _padded_size) {
_padded_size = *size + simdjson::SIMDJSON_PADDING;
_padding_buffer.resize(_padded_size);
}
// Ondemand values reference the input buffer. Keep the padded bytes in a member buffer until the
// current document is fully materialized.
std::memcpy(_padding_buffer.data(), _document_view.data(), *size);
// current document is fully materialized. File and pipe documents are already there, at offset
// 3 when a BOM was skipped, so the ranges may overlap.
if (_document_view.data() != _padding_buffer.data()) {
_grow_padding_buffer(*size);
std::memmove(_padding_buffer.data(), _document_view.data(), *size);
}
_original_doc_size = *size;
const std::string_view document(_padding_buffer.data(), *size);
const auto error =
_json_parser->iterate(std::string_view(_padding_buffer.data(), *size), _padded_size)
.get(_original_json_doc);
_json_parser->iterate(document, _padding_buffer.size()).get(_original_json_doc);
if (error != simdjson::error_code::SUCCESS) {
return Status::DataQualityError(
"Parse json data for JsonDoc failed. code: {}, error info: {}", error,
Expand Down
9 changes: 6 additions & 3 deletions be/src/format_v2/json/json_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,10 @@ class JsonReader final : public FileReader {
Status close() override;

#ifdef BE_TEST
size_t TEST_document_buffer_size() const { return _document_buffer.size(); }
bool TEST_document_in_padding_buffer() const {
return !_document_view.empty() && _document_view.data() == _padding_buffer.data();
}
size_t TEST_padding_buffer_size() const { return _padding_buffer.size(); }
#endif

private:
Expand Down Expand Up @@ -139,6 +142,8 @@ class JsonReader final : public FileReader {
// Chunked HTTP transfers concatenate their pipe buffers into one document.
Status _read_one_document(size_t* size, bool* eof);
Status _read_one_document_from_pipe(size_t* read_size);
// Grows `_padding_buffer` to hold `size` bytes plus simdjson padding, keeping its contents.
void _grow_padding_buffer(size_t size);
// Moves the logical document into a simdjson-padded buffer and creates an ondemand document.
Status _parse_next_json(size_t* size, bool* eof);
// Applies json_root and validates the object/array shape required by strip_outer_array.
Expand Down Expand Up @@ -223,11 +228,9 @@ class JsonReader final : public FileReader {
simdjson::ondemand::value _json_value;
simdjson::ondemand::array _array;
simdjson::ondemand::array_iterator _array_iter;
std::string _document_buffer;
std::string_view _document_view;
std::string _padding_buffer;
size_t _original_doc_size = 0;
size_t _padded_size = 1024 * 1024 * 8 + simdjson::SIMDJSON_PADDING;
SimdJsonReadContext _simdjson_context;
};

Expand Down
16 changes: 13 additions & 3 deletions be/test/format_v2/json/json_reader_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include <fstream>
#include <memory>
#include <string>
#include <string_view>
#include <thread>
#include <vector>

Expand Down Expand Up @@ -262,7 +263,8 @@ struct ReadResult {
bool eof = false;
size_t second_rows = 0;
bool second_eof = false;
size_t document_buffer_size = 0;
bool document_in_padding_buffer = false;
size_t padding_buffer_size = 0;
int64_t rejected_rows = 0;
std::map<LocalColumnId, std::vector<uint8_t>> missing_fields;
std::vector<ColumnDefinition> schema;
Expand Down Expand Up @@ -310,7 +312,8 @@ ReadResult read_once(const std::string& file_name, const std::string& content,

result.block = make_block(result.schema, requested_local_ids);
result.status = reader.get_block(&result.block, &result.rows, &result.eof);
result.document_buffer_size = reader.TEST_document_buffer_size();
result.document_in_padding_buffer = reader.TEST_document_in_padding_buffer();
result.padding_buffer_size = reader.TEST_padding_buffer_size();
result.rejected_rows = io_ctx->load_filtered_rows;
result.missing_fields = reader.missing_fields();
if (result.status.ok() && read_twice) {
Expand Down Expand Up @@ -687,7 +690,10 @@ TEST(JsonReaderTest, ReadsRequestedColumnsInFileScanRequestOrder) {
ASSERT_TRUE(result.second_status.ok()) << result.second_status.to_string();
EXPECT_EQ(result.second_rows, 0);
EXPECT_TRUE(result.second_eof);
EXPECT_EQ(result.document_buffer_size, 0);
EXPECT_FALSE(result.document_in_padding_buffer);
// The buffer fits the longest line instead of a fixed 8 MB.
EXPECT_EQ(result.padding_buffer_size,
std::string_view(R"({"id":1,"name":"alice"})").size() + simdjson::SIMDJSON_PADDING);
}

TEST(JsonReaderTest, HiveColumnLookupIsCaseInsensitiveWithoutNormalizedKeys) {
Expand Down Expand Up @@ -716,6 +722,10 @@ TEST(JsonReaderTest, ReadsSingleDocumentOuterArray) {
EXPECT_EQ(nullable_string_at(*result.block.get_by_position(1).column, 0), "carol");
EXPECT_EQ(nullable_int_at(*result.block.get_by_position(0).column, 1), 4);
EXPECT_EQ(nullable_string_at(*result.block.get_by_position(1).column, 1), "dave");
EXPECT_TRUE(result.document_in_padding_buffer);
EXPECT_EQ(result.padding_buffer_size,
std::string_view(R"([{"id":3,"name":"carol"},{"id":4,"name":"dave"}])").size() +
simdjson::SIMDJSON_PADDING);
}

TEST(JsonReaderTest, ReadsJsonRootByLine) {
Expand Down
Loading