diff --git a/.github/workflows/test-shared.yml b/.github/workflows/test-shared.yml index 73254d292ec1..a79e03df8663 100644 --- a/.github/workflows/test-shared.yml +++ b/.github/workflows/test-shared.yml @@ -31,7 +31,6 @@ on: - deps/ngtcp2/** - deps/openssl/*/** - deps/perfetto/** - - deps/simdjson/** - deps/sqlite/** - deps/uv/** - deps/uvwasi/** @@ -89,7 +88,6 @@ on: - deps/ngtcp2/** - deps/openssl/*/** - deps/perfetto/** - - deps/simdjson/** - deps/sqlite/** - deps/uv/** - deps/uvwasi/** diff --git a/.github/workflows/tools.yml b/.github/workflows/tools.yml index 15b225474078..b1f0e202c23f 100644 --- a/.github/workflows/tools.yml +++ b/.github/workflows/tools.yml @@ -26,6 +26,7 @@ on: - histogram - icu - inspector_protocol + - jsonifier - libffi - libuv - lief @@ -38,7 +39,6 @@ on: - perfetto - postject - root-certificates - - simdjson - sqlite - test426-fixtures - undici @@ -176,6 +176,14 @@ jobs: cat temp-output tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true rm temp-output + - id: jsonifier + subsystem: deps + label: dependencies + run: | + ./tools/dep_updaters/update-jsonifier.sh > temp-output + cat temp-output + tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true + rm temp-output - id: libuv subsystem: deps label: dependencies @@ -265,14 +273,6 @@ jobs: label: crypto, notable-change run: | node ./tools/dep_updaters/update-root-certs.mjs -v -f "$GITHUB_ENV" - - id: simdjson - subsystem: deps - label: dependencies - run: | - ./tools/dep_updaters/update-simdjson.sh > temp-output - cat temp-output - tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true - rm temp-output - id: sqlite subsystem: deps label: dependencies, sqlite diff --git a/LICENSE b/LICENSE index ec72f56e16fb..d7236d9d2182 100644 --- a/LICENSE +++ b/LICENSE @@ -1809,209 +1809,29 @@ The externally maintained libraries used by Node.js are: jloup@gzip.org madler@alumni.caltech.edu """ -- simdjson, located at deps/simdjson, is licensed as follows: +- Jsonifier, located at deps/jsonifier, is licensed as follows: """ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. + MIT License - Copyright 2018-2025 The simdjson authors + Copyright (c) 2026 Nihilai Collective Corp - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - http://www.apache.org/licenses/LICENSE-2.0 + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. """ - simdutf, located at deps/v8/third_party/simdutf, is licensed as follows: diff --git a/Makefile b/Makefile index cd732faeec2d..c6090f31525d 100644 --- a/Makefile +++ b/Makefile @@ -1310,7 +1310,6 @@ ifeq ($(SKIP_SHARED_DEPS), 1) find $(TARNAME)/deps/openssl -maxdepth 1 -type f ! -name 'nodejs-openssl.cnf' -exec $(RM) {} + find $(TARNAME)/deps/openssl -mindepth 1 -maxdepth 1 -type d -exec $(RM) -r {} + $(RM) -r $(TARNAME)/deps/perfetto - $(RM) -r $(TARNAME)/deps/simdjson $(RM) -r $(TARNAME)/deps/sqlite $(RM) -r $(TARNAME)/deps/uv $(RM) -r $(TARNAME)/deps/uvwasi diff --git a/benchmark/module/package-json-parse.js b/benchmark/module/package-json-parse.js new file mode 100644 index 000000000000..be5c6affdc93 --- /dev/null +++ b/benchmark/module/package-json-parse.js @@ -0,0 +1,82 @@ +'use strict'; +const common = require('../common.js'); +const assert = require('assert'); + +const bench = common.createBenchmark(main, { + size: ['minimal', 'typical', 'large'], + n: [1e5], +}, { flags: ['--expose-internals', '--no-warnings'] }); + +function makeMinimal() { + return { main: 'index.js' }; +} + +function makeTypical() { + return { + name: '@scope/typical-package', + version: '3.14.15', + description: 'A package with "escapes", unicode é中 and a\ttab', + type: 'module', + main: './dist/index.cjs', + exports: { + '.': { + types: './dist/index.d.ts', + import: './dist/index.mjs', + require: './dist/index.cjs', + }, + './utils': { import: './dist/utils.mjs', require: './dist/utils.cjs' }, + './package.json': './package.json', + }, + imports: { '#internal/*': './src/internal/*.js' }, + scripts: { build: 'tsc -p .', test: 'node --test', lint: 'eslint .' }, + keywords: ['json', 'parser', 'benchmark'], + private: false, + dependencies: { 'left-pad': '^1.3.0', 'lodash': '^4.17.21' }, + }; +} + +function makeLarge() { + const pkg = makeTypical(); + pkg.dependencies = {}; + pkg.devDependencies = {}; + for (let i = 0; i < 250; i++) { + pkg.dependencies[`dependency-number-${i}`] = `^${i % 17}.${i % 7}.${i}`; + pkg.devDependencies[`@types/dev-dependency-${i}`] = `~${i}.0.0`; + } + for (let i = 0; i < 100; i++) { + pkg.exports[`./feature-${i}`] = { + types: `./dist/feature-${i}.d.ts`, + import: `./dist/feature-${i}.mjs`, + require: `./dist/feature-${i}.cjs`, + }; + } + pkg.contributors = Array.from({ length: 50 }, (_, i) => ({ + name: `Contributor ${i}`, + email: `contributor${i}@example.com`, + })); + return pkg; +} + +const payloads = { + minimal: makeMinimal, + typical: makeTypical, + large: makeLarge, +}; + +function main({ size, n }) { + const { internalBinding } = require('internal/test/binding'); + const { parsePackageJSON } = internalBinding('modules'); + const source = Buffer.from(JSON.stringify(payloads[size](), null, 2)); + const jsonPath = `/benchmark/${size}/package.json`; + + const expected = parsePackageJSON(source, jsonPath); + assert.ok(Array.isArray(expected)); + + let result; + bench.start(); + for (let i = 0; i < n; i++) { + result = parsePackageJSON(source, jsonPath); + } + bench.end(n); + assert.deepStrictEqual(result, expected); +} diff --git a/configure.py b/configure.py index cecb6ae1b39e..97ef3e42670d 100755 --- a/configure.py +++ b/configure.py @@ -600,28 +600,6 @@ dest='shared_zlib_libpath', help='a directory to search for the shared zlib DLL') -shared_optgroup.add_argument('--shared-simdjson', - action='store_true', - dest='shared_simdjson', - default=None, - help='link to a shared simdjson DLL instead of static linking') - -shared_optgroup.add_argument('--shared-simdjson-includes', - action='store', - dest='shared_simdjson_includes', - help='directory containing simdjson header files') - -shared_optgroup.add_argument('--shared-simdjson-libname', - action='store', - dest='shared_simdjson_libname', - help='alternative lib name to link to [default: %(default)s]') - -shared_optgroup.add_argument('--shared-simdjson-libpath', - action='store', - dest='shared_simdjson_libpath', - help='a directory to search for the shared simdjson DLL') - - shared_optgroup.add_argument('--shared-simdutf', action='store_true', dest='shared_simdutf', @@ -3041,7 +3019,6 @@ def make_bin_override(): 'absl_variant', ]) configure_library('ada', output) -configure_library('simdjson', output) configure_library('simdutf', output) configure_library('brotli', output, pkgname=['libbrotlidec', 'libbrotlienc']) configure_library('cares', output, pkgname='libcares') diff --git a/deps/jsonifier/LICENSE b/deps/jsonifier/LICENSE new file mode 100644 index 000000000000..c711177f1d31 --- /dev/null +++ b/deps/jsonifier/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Nihilai Collective Corp + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/deps/jsonifier/THIRD_PARTY_LICENSES.md b/deps/jsonifier/THIRD_PARTY_LICENSES.md new file mode 100644 index 000000000000..106bd7bb1962 --- /dev/null +++ b/deps/jsonifier/THIRD_PARTY_LICENSES.md @@ -0,0 +1,33 @@ +Third-Party Licenses + +This project bundles the following third-party components. + +fast_float (fast_float) + +MIT License + +Copyright (c) 2021 The jsonifier_fast_float authors + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +(Also available under the Apache License 2.0 or the Boost Software License 1.0 — see upstream repository for full text of those alternatives.) + +zmij + +Copyright (c) 2025 - present, Victor Zverovich + +Distributed under the MIT license (see below) or alternatively the Boost Software License, Version 1.0. + +https://github.com/vitaut/zmij/ + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/deps/jsonifier/VERSION b/deps/jsonifier/VERSION new file mode 100644 index 000000000000..6d7de6e6abef --- /dev/null +++ b/deps/jsonifier/VERSION @@ -0,0 +1 @@ +1.0.2 diff --git a/deps/jsonifier/include/jsonifier b/deps/jsonifier/include/jsonifier new file mode 100644 index 000000000000..3d345fcee790 --- /dev/null +++ b/deps/jsonifier/include/jsonifier @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier + */ + +#pragma once + +#include + +#if defined(reinterpret_cast) + #undef reinterpret_cast +#endif + +#if defined(const_cast) + #undef const_cast +#endif + +#if defined(dynamic_cast) + #undef dynamic_cast +#endif + +#if defined(memcpy) + #undef memcpy +#endif + +#if defined(memmove) + #undef memmove +#endif diff --git a/deps/jsonifier/include/jsonifier-incl/containers/allocator.hpp b/deps/jsonifier/include/jsonifier-incl/containers/allocator.hpp new file mode 100644 index 000000000000..8699b3149c3d --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/containers/allocator.hpp @@ -0,0 +1,197 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/containers/allocator.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::internal { + + template JSONIFIER_INLINE constexpr value_type roundDownToMultiple(value_type value) noexcept { + if constexpr ((multiple & (multiple - 1)) == 0) { + constexpr auto notMulSub1{ ~(multiple - 1) }; + return value & notMulSub1; + } else { + if constexpr (std::is_signed_v) { + return value >= 0 ? (value / multiple) * multiple : ((value - multiple + 1) / multiple) * multiple; + } else { + return (value / multiple) * multiple; + } + } + } + + template JSONIFIER_INLINE constexpr value_type roundUpToMultiple(value_type value) noexcept { + if constexpr ((multiple & (multiple - 1)) == 0) { + constexpr auto mulSub1{ multiple - 1 }; + constexpr auto notMulSub1{ ~mulSub1 }; + return (value + static_cast(mulSub1)) & static_cast(notMulSub1); + } else { + const auto remainder = value % multiple; + return remainder == 0 ? value : value + (multiple - remainder); + } + } + + enum class allocated_memory_types : uint8_t { + huge_page, + mmap, + standard_page, + standard, + }; + + template struct alloc_wrapper { + using value_type = value_type_new; + using size_type = uint64_t; + using difference_type = ptrdiff_t; + using pointer = value_type*; + using const_pointer = const value_type*; + static constexpr uint64_t alignment = (alignof(value_type_new) > simdBytesPerRegister) ? alignof(value_type_new) : simdBytesPerRegister; + + struct allocation_header { + size_type totalBytes{}; + allocated_memory_types allocated_memory_type{}; + }; + + static constexpr uint64_t headerSize = roundUpToMultiple(sizeof(allocation_header)); + static constexpr uint64_t hugePageSize = 2 * 1024 * 1024ULL; + static constexpr uint64_t hugePageThreshold = hugePageSize * 2; + + using allocator_traits = std::allocator_traits>; + + template struct rebind { + using other = alloc_wrapper; + }; + + JSONIFIER_INLINE constexpr alloc_wrapper() noexcept = default; + + template JSONIFIER_INLINE constexpr alloc_wrapper(const alloc_wrapper&) noexcept { + } + + JSONIFIER_INLINE static pointer finalize(void* base, allocated_memory_types allocated_memory_type, size_type totalBytes) noexcept { + allocation_header* header = static_cast(base); + header->totalBytes = totalBytes; + header->allocated_memory_type = allocated_memory_type; + return std::bit_cast(static_cast(base) + headerSize); + } + + template JSONIFIER_INLINE static void construct(pointer p, arg_types&&... args) noexcept { + new (p) value_type(internal::forward(args)...); + } + + JSONIFIER_INLINE static pointer allocate(size_type count) noexcept { + if (count == 0) [[unlikely]] { + return nullptr; + } + if (count > maxSize()) [[unlikely]] { + return nullptr; + } + const size_type bytes = count * sizeof(value_type) + headerSize; + const size_type alignedBytes = roundUpToMultiple(bytes); + if (alignedBytes >= hugePageThreshold) { + [[maybe_unused]] const size_type hpBytes = roundUpToMultiple(alignedBytes); +#if JSONIFIER_PLATFORM_WINDOWS + void* p = VirtualAlloc(nullptr, hpBytes, MEM_RESERVE | MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE); + if (p != nullptr) { + return finalize(p, allocated_memory_types::huge_page, hpBytes); + } + p = VirtualAlloc(nullptr, hpBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + if (p != nullptr) { + return finalize(p, allocated_memory_types::standard_page, hpBytes); + } +#elif JSONIFIER_PLATFORM_LINUX + void* p = mmap(nullptr, hpBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); + if (p != MAP_FAILED) { + return finalize(p, allocated_memory_types::huge_page, hpBytes); + } + p = mmap(nullptr, hpBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p != MAP_FAILED) { + madvise(p, hpBytes, MADV_HUGEPAGE); + return finalize(p, allocated_memory_types::mmap, hpBytes); + } +#elif JSONIFIER_PLATFORM_MAC || JSONIFIER_PLATFORM_ANDROID + void* p = mmap(nullptr, hpBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p != MAP_FAILED) { + return finalize(p, allocated_memory_types::mmap, hpBytes); + } +#endif + } +#if (JSONIFIER_PLATFORM_WINDOWS || JSONIFIER_PLATFORM_LINUX) && JSONIFIER_ANY_AVX + void* p = _mm_malloc(alignedBytes, alignment); +#elif JSONIFIER_PLATFORM_WINDOWS + void* p = _aligned_malloc(alignedBytes, alignment); +#else + void* p = aligned_alloc(alignment, alignedBytes); +#endif + if (p == nullptr) [[unlikely]] { + return nullptr; + } + return finalize(p, allocated_memory_types::standard, alignedBytes); + } + + static consteval size_type maxSize() noexcept { + return (static_cast(-1) - headerSize - (alignment - 1)) / sizeof(value_type); + } + + JSONIFIER_INLINE constexpr bool operator!=(const alloc_wrapper& other) const noexcept { + return !(*this == other); + } + + JSONIFIER_INLINE static void deallocate(pointer p, size_type) noexcept { + if (p) [[likely]] { + void* base = std::bit_cast(p) - headerSize; + auto* header = static_cast(base); + [[maybe_unused]] size_type totalBytes = header->totalBytes; + switch (static_cast(header->allocated_memory_type)) { + case static_cast(allocated_memory_types::huge_page): { +#if JSONIFIER_PLATFORM_WINDOWS + VirtualFree(base, 0, MEM_RELEASE); +#elif JSONIFIER_PLATFORM_LINUX + munmap(base, totalBytes); +#endif + break; + } + case static_cast(allocated_memory_types::mmap): { +#if JSONIFIER_PLATFORM_LINUX || JSONIFIER_PLATFORM_MAC || JSONIFIER_PLATFORM_ANDROID + munmap(base, totalBytes); +#endif + break; + } + case static_cast(allocated_memory_types::standard_page): { +#if JSONIFIER_PLATFORM_WINDOWS + VirtualFree(base, 0, MEM_RELEASE); +#elif JSONIFIER_PLATFORM_LINUX || JSONIFIER_PLATFORM_MAC + munmap(base, totalBytes); +#endif + break; + } + case static_cast(allocated_memory_types::standard): { +#if (JSONIFIER_PLATFORM_WINDOWS || JSONIFIER_PLATFORM_LINUX) && JSONIFIER_ANY_AVX + _mm_free(base); +#elif JSONIFIER_PLATFORM_WINDOWS + _aligned_free(base); +#else + free(base); +#endif + break; + } + default: { + break; + } + } + } + } + + JSONIFIER_INLINE constexpr bool operator==(const alloc_wrapper&) const noexcept { + return true; + } + + JSONIFIER_INLINE static constexpr void destroy(pointer) noexcept { + } + + static_assert(alignment >= alignof(allocation_header), "alignment must cover allocation_header's alignment requirement."); + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/containers/array.hpp b/deps/jsonifier/include/jsonifier-incl/containers/array.hpp new file mode 100644 index 000000000000..6a7d045aad92 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/containers/array.hpp @@ -0,0 +1,171 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/containers/array.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::internal { + + template struct array { + public: + static_assert(std::is_object_v, "The C++ Standard forbids containers of non-object types because of [container.requirements]."); + + using value_type = value_type_new; + using size_type = uint64_t; + using difference_type = ptrdiff_t; + using reference = value_type&; + using r_reference = value_type&&; + using const_reference = const value_type&; + using const_r_reference = const value_type&&; + using pointer = value_type*; + using const_pointer = const value_type*; + using iterator = sized_iterator; + using const_iterator = sized_iterator; + + template index_type> JSONIFIER_INLINE constexpr const_r_reference operator[](index_type position) const&& noexcept { + return static_cast(dataVal[static_cast(position)]); + } + + template index_type> JSONIFIER_INLINE constexpr const_reference operator[](index_type position) const& noexcept { + return static_cast(dataVal[static_cast(position)]); + } + + template JSONIFIER_INLINE constexpr const_r_reference operator[](tag_type position) const&& noexcept { + static_assert(static_cast(position) < sizeNew, "Sorry, but that index is out of bounds!"); + return static_cast(dataVal[static_cast(position)]); + } + + template JSONIFIER_INLINE constexpr const_reference operator[](tag_type position) const& noexcept { + static_assert(static_cast(position) < sizeNew, "Sorry, but that index is out of bounds!"); + return static_cast(dataVal[static_cast(position)]); + } + + template index_type> JSONIFIER_INLINE constexpr r_reference operator[](index_type position) && noexcept { + return static_cast(dataVal[static_cast(position)]); + } + + template index_type> JSONIFIER_INLINE constexpr reference operator[](index_type position) & noexcept { + return static_cast(dataVal[static_cast(position)]); + } + + template JSONIFIER_INLINE constexpr r_reference operator[](tag_type position) && noexcept { + static_assert(static_cast(position) < sizeNew, "Sorry, but that index is out of bounds!"); + return static_cast(dataVal[static_cast(position)]); + } + + template JSONIFIER_INLINE constexpr reference operator[](tag_type position) & noexcept { + static_assert(static_cast(position) < sizeNew, "Sorry, but that index is out of bounds!"); + return static_cast(dataVal[static_cast(position)]); + } + + JSONIFIER_INLINE constexpr void swap(array& other) noexcept(std::is_nothrow_swappable::value) { + std::swap_ranges(dataVal, dataVal + sizeNew, other.dataVal); + } + + JSONIFIER_INLINE constexpr friend bool operator==(const array& lhs, const array& rhs) noexcept { + for (uint64_t x = 0; x < sizeNew; ++x) { + if (lhs[x] != rhs[x]) { + return false; + } + } + return true; + } + + JSONIFIER_INLINE constexpr friend bool operator!=(const array& lhs, const array& rhs) noexcept { + return !(lhs == rhs); + } + + JSONIFIER_INLINE constexpr const_reference front() const noexcept JSONIFIER_LIFETIME_BOUND { + return dataVal[0]; + } + + JSONIFIER_INLINE constexpr const_iterator begin() const noexcept JSONIFIER_LIFETIME_BOUND { + return const_iterator{ dataVal }; + } + + JSONIFIER_INLINE constexpr const_reference back() const noexcept JSONIFIER_LIFETIME_BOUND { + return dataVal[sizeNew - 1]; + } + + JSONIFIER_INLINE constexpr const_iterator end() const noexcept JSONIFIER_LIFETIME_BOUND { + return const_iterator{ dataVal + sizeNew }; + } + + JSONIFIER_INLINE constexpr reference front() noexcept JSONIFIER_LIFETIME_BOUND { + return dataVal[0]; + } + + JSONIFIER_INLINE constexpr iterator begin() noexcept JSONIFIER_LIFETIME_BOUND { + return iterator{ dataVal }; + } + + JSONIFIER_INLINE constexpr reference back() noexcept JSONIFIER_LIFETIME_BOUND { + return dataVal[sizeNew - 1]; + } + + JSONIFIER_INLINE constexpr iterator end() noexcept JSONIFIER_LIFETIME_BOUND { + return iterator{ dataVal + sizeNew }; + } + + JSONIFIER_INLINE constexpr void fill(const value_type& value) noexcept { + std::fill_n(dataVal, sizeNew, value); + } + + JSONIFIER_INLINE constexpr const_pointer data() const noexcept { + return dataVal; + } + + JSONIFIER_INLINE constexpr size_type size() const noexcept { + return sizeNew; + } + + JSONIFIER_INLINE constexpr bool empty() const noexcept { + return false; + } + + JSONIFIER_INLINE constexpr pointer data() noexcept { + return dataVal; + } + + value_type dataVal[sizeNew]; + }; + + template array(first, rest...) -> array, 1 + sizeof...(rest)>; + + template array(array) -> array, size>; + + template struct array { + public: + using value_type = value_type_new; + using size_type = uint64_t; + using difference_type = ptrdiff_t; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; + using iterator = sized_iterator; + using const_iterator = sized_iterator; + + JSONIFIER_INLINE constexpr friend bool operator==(const array&, const array&) noexcept { + return true; + } + + JSONIFIER_INLINE constexpr friend bool operator!=(const array&, const array&) noexcept { + return false; + } + + JSONIFIER_INLINE constexpr size_type size() const noexcept { + return 0; + } + + JSONIFIER_INLINE constexpr bool empty() const noexcept { + return true; + } + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/containers/iterator.hpp b/deps/jsonifier/include/jsonifier-incl/containers/iterator.hpp new file mode 100644 index 000000000000..c57781c8b01f --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/containers/iterator.hpp @@ -0,0 +1,162 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/containers/iterator.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::internal { + + template struct sized_iterator { + using iterator_concept = std::contiguous_iterator_tag; + using iterator_category = std::random_access_iterator_tag; + using element_type = value_type_new; + using value_type = value_type_new; + using difference_type = ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + JSONIFIER_INLINE constexpr sized_iterator() noexcept : ptr{ nullptr } { + } + + JSONIFIER_INLINE constexpr sized_iterator(pointer ptrNew) noexcept : ptr{ ptrNew } { + } + + JSONIFIER_INLINE friend constexpr sized_iterator operator+(const difference_type offSet, sized_iterator next) noexcept { + next += offSet; + return next; + } + + JSONIFIER_INLINE constexpr difference_type operator-(const sized_iterator& other) const noexcept { + return ptr - other.ptr; + } + + JSONIFIER_INLINE constexpr sized_iterator operator+(const difference_type offSet) const noexcept { + sized_iterator temp = *this; + temp += offSet; + return temp; + } + + JSONIFIER_INLINE constexpr sized_iterator operator-(const difference_type offSet) const noexcept { + sized_iterator temp = *this; + temp -= offSet; + return temp; + } + + JSONIFIER_INLINE constexpr reference operator[](const difference_type offSet) const noexcept { + return *(*this + offSet); + } + + JSONIFIER_INLINE constexpr sized_iterator& operator-=(const difference_type offSet) noexcept { + return *this += -offSet; + } + + JSONIFIER_INLINE constexpr sized_iterator& operator+=(const difference_type offSet) noexcept { + ptr += offSet; + return *this; + } + + JSONIFIER_INLINE constexpr bool operator!=(const sized_iterator& other) const noexcept { + return !(*this == other); + } + + JSONIFIER_INLINE constexpr bool operator==(const sized_iterator& other) const noexcept { + return ptr == other.ptr; + } + + JSONIFIER_INLINE constexpr bool operator<=(const sized_iterator& other) const noexcept { + return !(other < *this); + } + + JSONIFIER_INLINE constexpr bool operator>=(const sized_iterator& other) const noexcept { + return !(*this < other); + } + + JSONIFIER_INLINE constexpr bool operator<(const sized_iterator& other) const noexcept { + return ptr < other.ptr; + } + + JSONIFIER_INLINE constexpr bool operator>(const sized_iterator& other) const noexcept { + return other < *this; + } + + JSONIFIER_INLINE constexpr sized_iterator operator++(int32_t) noexcept { + sized_iterator temp = *this; + ++*this; + return temp; + } + + JSONIFIER_INLINE constexpr sized_iterator operator--(int32_t) noexcept { + sized_iterator temp = *this; + --*this; + return temp; + } + + JSONIFIER_INLINE constexpr sized_iterator& operator++() noexcept { + ++ptr; + return *this; + } + + JSONIFIER_INLINE constexpr sized_iterator& operator--() noexcept { + --ptr; + return *this; + } + + JSONIFIER_INLINE constexpr reference operator*() const noexcept { + return *ptr; + } + + JSONIFIER_INLINE constexpr pointer operator->() const noexcept { + return ptr; + } + + pointer ptr; + }; + + template struct sized_iterator { + using iterator_concept = std::contiguous_iterator_tag; + using iterator_category = std::random_access_iterator_tag; + using element_type = value_type_new; + using value_type = value_type_new; + using difference_type = ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + JSONIFIER_INLINE constexpr sized_iterator() noexcept { + } + + JSONIFIER_INLINE constexpr sized_iterator(std::nullptr_t) noexcept { + } + + JSONIFIER_INLINE constexpr bool operator<=(const sized_iterator& other) const noexcept { + return !(other < *this); + } + + JSONIFIER_INLINE constexpr bool operator>=(const sized_iterator& other) const noexcept { + return !(*this < other); + } + + JSONIFIER_INLINE constexpr bool operator!=(const sized_iterator&) const noexcept { + return false; + } + + JSONIFIER_INLINE constexpr bool operator==(const sized_iterator&) const noexcept { + return true; + } + + JSONIFIER_INLINE constexpr bool operator<(const sized_iterator&) const noexcept { + return false; + } + + JSONIFIER_INLINE constexpr bool operator>(const sized_iterator&) const noexcept { + return false; + } + }; + + template using basic_iterator = sized_iterator; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/containers/tuple.hpp b/deps/jsonifier/include/jsonifier-incl/containers/tuple.hpp new file mode 100644 index 000000000000..96aca0bd72e2 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/containers/tuple.hpp @@ -0,0 +1,438 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/containers/tuple.hpp + */ +#pragma once + +#include + +namespace jsonifier::internal { + + template struct type_list_elem { + using value_type = value_type_new; + static constexpr uint64_t index{ indexNew }; + template friend struct type_list_element; + + JSONIFIER_INLINE constexpr decltype(auto) operator[](tag) const&& noexcept { + return static_cast(value); + } + + JSONIFIER_INLINE constexpr decltype(auto) operator[](tag) const& noexcept { + return static_cast(value); + } + + JSONIFIER_INLINE constexpr decltype(auto) operator[](tag) && noexcept { + return static_cast(value); + } + + JSONIFIER_INLINE constexpr decltype(auto) operator[](tag) & noexcept { + return static_cast(value); + } + + static consteval type_list_elem getForType(tag) noexcept { + return type_list_elem{}; + } + + JSONIFIER_TUPLET_NO_UNIQUE_ADDRESS value_type value; + }; + + template struct type_list_elem : public remove_const_t> { + using value_type = value_type_new; + static constexpr uint64_t index{ indexNew }; + template friend struct type_list_element; + + JSONIFIER_INLINE constexpr decltype(auto) operator[](tag) const&& noexcept JSONIFIER_LIFETIME_BOUND { + return static_cast(*this); + } + + JSONIFIER_INLINE constexpr decltype(auto) operator[](tag) const& noexcept JSONIFIER_LIFETIME_BOUND { + return static_cast(*this); + } + + JSONIFIER_INLINE constexpr decltype(auto) operator[](tag) && noexcept JSONIFIER_LIFETIME_BOUND { + return static_cast(*this); + } + + JSONIFIER_INLINE constexpr decltype(auto) operator[](tag) & noexcept JSONIFIER_LIFETIME_BOUND { + return static_cast(*this); + } + + static consteval type_list_elem getForType(tag) noexcept { + return type_list_elem{}; + } + }; + + template struct type_list_impl : public value_types... { + static constexpr uint64_t size{ sizeof...(value_types) }; + using value_types::operator[]...; + using value_types::getForType...; + }; + + template struct tuple_type_list; + + template struct tuple_type_list, value_types...> { + using type = type_list_impl...>; + }; + + template using type_list_t = typename tuple_type_list, value_types...>::type; + + template struct type_list_element { + using type = typename decltype(remove_pointer_t>::getForType(tag{}))::value_type; + }; + + template using type_list_element_t = type_list_element::type; + + template using tuple_element_t = type_list_element_t; + + struct eq_op { + template value_type_02> + JSONIFIER_INLINE static constexpr bool impl(value_type_01&& val01, value_type_02&& val02) noexcept { + return std::forward(val01) == std::forward(val02); + } + }; + + struct neq_op { + template value_type_02> + JSONIFIER_INLINE static constexpr bool impl(value_type_01&& val01, value_type_02&& val02) noexcept { + return std::forward(val01) != std::forward(val02); + } + }; + + struct lt_op { + template value_type_02> + JSONIFIER_INLINE static constexpr bool impl(value_type_01&& val01, value_type_02&& val02) noexcept { + return std::forward(val01) < std::forward(val02); + } + }; + + struct lte_op { + template value_type_02> + JSONIFIER_INLINE static constexpr bool impl(value_type_01&& val01, value_type_02&& val02) noexcept { + return std::forward(val01) <= std::forward(val02); + } + }; + + struct gt_op { + template value_type_02> + JSONIFIER_INLINE static constexpr bool impl(value_type_01&& val01, value_type_02&& val02) noexcept { + return std::forward(val01) > std::forward(val02); + } + }; + + struct gte_op { + template value_type_02> + JSONIFIER_INLINE static constexpr bool impl(value_type_01&& val01, value_type_02&& val02) noexcept { + return std::forward(val01) >= std::forward(val02); + } + }; + + struct ss_op { + template value_type_02> + JSONIFIER_INLINE static constexpr auto impl(value_type_01&& val01, value_type_02&& val02) noexcept { + return std::forward(val01) <=> std::forward(val02); + } + }; + + template struct comparison_op; + + template struct comparison_op, eq_op> { + template typename tuple_type_01, typename... value_types_01, template typename tuple_type_02, typename... value_types_02> + JSONIFIER_INLINE static constexpr bool impl(const tuple_type_01& t1, const tuple_type_02& t2) noexcept { + return (eq_op::impl(t1[tag{}], t2[tag{}]) && ...); + } + }; + + template struct comparison_op, neq_op> { + template typename tuple_type_01, typename... value_types_01, template typename tuple_type_02, typename... value_types_02> + JSONIFIER_INLINE static constexpr bool impl(const tuple_type_01& t1, const tuple_type_02& t2) noexcept { + return !(t1 == t2); + } + }; + + template struct ordering_accumulator { + JSONIFIER_INLINE constexpr ordering_accumulator& operator=(result_type_new newValue) noexcept { + value = newValue; + return *this; + } + + JSONIFIER_INLINE constexpr operator bool() const noexcept { + return std::is_eq(value); + } + + result_type_new value{ result_type_new::equivalent }; + }; + + template struct comparison_op, ss_op> { + template typename tuple_type_01, typename... value_types_01, template typename tuple_type_02, typename... value_types_02> + JSONIFIER_INLINE static constexpr auto impl(const tuple_type_01& t1, const tuple_type_02& t2) noexcept + -> std::common_comparison_category_t() <=> std::declval())...> { + using result_type = std::common_comparison_category_t() <=> std::declval())...>; + ordering_accumulator result{}; + ((result = ss_op::impl(t1[tag{}], t2[tag{}])) && ...); + return result.value; + } + }; + + template struct comparison_op, lt_op> { + template typename tuple_type_01, typename... value_types_01, template typename tuple_type_02, typename... value_types_02> + JSONIFIER_INLINE static constexpr bool impl(const tuple_type_01& t1, const tuple_type_02& t2) noexcept { + bool result = false; + (((result = lt_op::impl(t1[tag{}], t2[tag{}])) || (!lt_op::impl(t2[tag{}], t1[tag{}]))) && ...); + return result; + } + }; + + template struct comparison_op, lte_op> { + template typename tuple_type_01, typename... value_types_01, template typename tuple_type_02, typename... value_types_02> + JSONIFIER_INLINE static constexpr bool impl(const tuple_type_01& t1, const tuple_type_02& t2) noexcept { + return !(t2 < t1); + } + }; + + template struct comparison_op, gt_op> { + template typename tuple_type_01, typename... value_types_01, template typename tuple_type_02, typename... value_types_02> + JSONIFIER_INLINE static constexpr bool impl(const tuple_type_01& t1, const tuple_type_02& t2) noexcept { + bool result = false; + (((result = gt_op::impl(t1[tag{}], t2[tag{}])) || (!gt_op::impl(t2[tag{}], t1[tag{}]))) && ...); + return result; + } + }; + + template struct comparison_op, gte_op> { + template typename tuple_type_01, typename... value_types_01, template typename tuple_type_02, typename... value_types_02> + JSONIFIER_INLINE static constexpr bool impl(const tuple_type_01& t1, const tuple_type_02& t2) noexcept { + return !(t1 < t2); + } + }; + + template struct tuple; + + template struct tuple_size; + + template struct tuple : type_list_t { + static constexpr uint64_t size{ sizeof...(value_types) }; + + template JSONIFIER_INLINE constexpr decltype(auto) operator<=>(const tuple& other) const noexcept { + static_assert(sizeof...(other_types) == size, "Sorry, but these tuples must be equal in size to be compared!"); + return comparison_op, ss_op>::impl(*this, other); + } + + template JSONIFIER_INLINE constexpr bool operator!=(const tuple& other) const noexcept { + static_assert(sizeof...(other_types) == size, "Sorry, but these tuples must be equal in size to be compared!"); + return comparison_op, neq_op>::impl(*this, other); + } + + template JSONIFIER_INLINE constexpr bool operator>=(const tuple& other) const noexcept { + static_assert(sizeof...(other_types) == size, "Sorry, but these tuples must be equal in size to be compared!"); + return comparison_op, gte_op>::impl(*this, other); + } + + template JSONIFIER_INLINE constexpr bool operator<=(const tuple& other) const noexcept { + static_assert(sizeof...(other_types) == size, "Sorry, but these tuples must be equal in size to be compared!"); + return comparison_op, lte_op>::impl(*this, other); + } + + template JSONIFIER_INLINE constexpr bool operator==(const tuple& other) const noexcept { + static_assert(sizeof...(other_types) == size, "Sorry, but these tuples must be equal in size to be compared!"); + return comparison_op, eq_op>::impl(*this, other); + } + + template JSONIFIER_INLINE constexpr bool operator>(const tuple& other) const noexcept { + static_assert(sizeof...(other_types) == size, "Sorry, but these tuples must be equal in size to be compared!"); + return comparison_op, gt_op>::impl(*this, other); + } + + template JSONIFIER_INLINE constexpr bool operator<(const tuple& other) const noexcept { + static_assert(sizeof...(other_types) == size, "Sorry, but these tuples must be equal in size to be compared!"); + return comparison_op, lt_op>::impl(*this, other); + } + }; + + template<> struct tuple<> : type_list_t<> { + static constexpr uint64_t size{ 0 }; + + template JSONIFIER_INLINE constexpr auto operator<=>(const tuple&) const noexcept { + static_assert(sizeof...(other_types) == size, "Sorry, but these tuples must be equal in size to be compared!"); + return std::strong_ordering::equal; + } + + template JSONIFIER_INLINE constexpr bool operator!=(const tuple&) const noexcept { + static_assert(sizeof...(other_types) == size, "Sorry, but these tuples must be equal in size to be compared!"); + return false; + } + + template JSONIFIER_INLINE constexpr bool operator==(const tuple&) const noexcept { + static_assert(sizeof...(other_types) == size, "Sorry, but these tuples must be equal in size to be compared!"); + return true; + } + + template JSONIFIER_INLINE constexpr bool operator>=(const tuple&) const noexcept { + static_assert(sizeof...(other_types) == size, "Sorry, but these tuples must be equal in size to be compared!"); + return true; + } + + template JSONIFIER_INLINE constexpr bool operator<=(const tuple&) const noexcept { + static_assert(sizeof...(other_types) == size, "Sorry, but these tuples must be equal in size to be compared!"); + return true; + } + + template JSONIFIER_INLINE constexpr bool operator>(const tuple&) const noexcept { + static_assert(sizeof...(other_types) == size, "Sorry, but these tuples must be equal in size to be compared!"); + return false; + } + + template JSONIFIER_INLINE constexpr bool operator<(const tuple&) const noexcept { + static_assert(sizeof...(other_types) == size, "Sorry, but these tuples must be equal in size to be compared!"); + return false; + } + }; + + template struct tuple_size> : public integral_constant {}; + + template struct tuple_size> : public integral_constant {}; + + template inline constexpr uint64_t tupleSizeOf = tuple_size>::value; + + template tuple(value_types&&...) -> tuple; + + template + requires(specialization_of_types, tuple>) + JSONIFIER_INLINE constexpr decltype(auto) getBecauseOtherLibAuthorsResolve(tuple_type&& t JSONIFIER_LIFETIME_BOUND) noexcept { + return std::forward(t)[tag{}]; + } + + struct completion_signal { + consteval operator bool() const noexcept { + return notDone; + } + + uint64_t index{}; + bool notDone{ true }; + }; + + template struct index_tag { + using element_type = type_list_element_t; + static constexpr bool isNotMatch = !std::is_same_v, remove_cvref_t>; + static constexpr uint64_t index{ isNotMatch ? std::numeric_limits::max() : indexNew }; + static constexpr completion_signal value{ index, isNotMatch }; + }; + + template struct index_finder; + + template struct index_finder> { + static consteval uint64_t getIndex() noexcept { + completion_signal result{}; + ((result = index_tag::value) && ...); + return result.index; + } + + static constexpr uint64_t index{ getIndex() }; + + static_assert(index != std::numeric_limits::max(), "Sorry, but that type does not appear to exist in this tuple!"); + }; + + template JSONIFIER_INLINE constexpr decltype(auto) get(tuple_type&& tupleVal) noexcept { + constexpr uint64_t index = index_finder::size>>::index; + return std::forward(tupleVal)[tag{}]; + } + + template JSONIFIER_INLINE constexpr auto makeTuple(types&&... args) noexcept { + using base = type_list_t; + return tuple{ base{ { std::forward(args) }... } }; + } + + template struct join_tuples; + + template struct join_tuples, tuple, rest_types...> { + using type = typename join_tuples, rest_types...>::type; + }; + + template struct join_tuples> { + using type = tuple; + }; + + template<> struct join_tuples<> { + using type = tuple<>; + }; + + template struct join_tuples, tuple> { + using type = tuple; + }; + + template using join_tuples_t = typename join_tuples::type; + + template struct tuple_cat_impl { + using lists_type = type_list_t; + static constexpr uint64_t total{ (remove_reference_t::size + ...) }; + using result_type = join_tuples_t...>; + using lists_tuple_type = type_list_t*...>; + + struct tuple_cat_index_map { + uint64_t localIdx[total > 0 ? total : 1]{}; + uint64_t listIdx[total > 0 ? total : 1]{}; + }; + + template struct tuple_cat_impl_internal; + + template struct tuple_cat_impl_internal> { + template JSONIFIER_INLINE static constexpr decltype(auto) impl(list_type&& lists) noexcept { + return result_type{ { { getIndividualElement(std::forward(lists)) }... } }; + } + }; + + template JSONIFIER_INLINE static constexpr decltype(auto) getIndividualElement(list_type&& list) noexcept { + constexpr uint64_t listIdx = map.listIdx[index]; + constexpr uint64_t localIdx = map.localIdx[index]; + auto* tuplePtr = list[tag{}]; + using source_type = type_list_element_t; + if constexpr (std::is_lvalue_reference_v) { + return (*tuplePtr)[tag{}]; + } else { + return internal::move((*tuplePtr)[tag{}]); + } + } + + template JSONIFIER_INLINE static constexpr decltype(auto) impl(tuple_list_types&... listVals) noexcept { + return tuple_cat_impl_internal>::impl(lists_tuple_type{ { { &listVals } }... }); + } + + static consteval tuple_cat_index_map getMapValues() noexcept { + tuple_cat_index_map m{}; + if constexpr (total > 0) { + constexpr uint64_t sizes[]{ remove_reference_t::size... }; + uint64_t g{}; + for (uint64_t i = 0; i < sizeof...(list_types); ++i) { + for (uint64_t j = 0; j < sizes[i]; ++j, ++g) { + m.listIdx[g] = i; + m.localIdx[g] = j; + } + } + } + return m; + } + + static constexpr auto map{ getMapValues() }; + }; + + template JSONIFIER_INLINE constexpr decltype(auto) tupleCat(tuple_list_types&&... listVals) noexcept { + return tuple_cat_impl::impl(listVals...); + } +} + +namespace std { + + template struct tuple_element> { + using type = jsonifier::internal::tuple_element_t>; + }; + + template struct tuple_size> + : public jsonifier::internal::integral_constant::size> {}; + + template + requires(jsonifier::internal::specialization_of_types, jsonifier::internal::tuple>) + JSONIFIER_INLINE constexpr decltype(auto) get(tuple_type&& t) noexcept { + return jsonifier::internal::getBecauseOtherLibAuthorsResolve(std::forward(t)); + } +} diff --git a/deps/jsonifier/include/jsonifier-incl/core/config.hpp b/deps/jsonifier/include/jsonifier-incl/core/config.hpp new file mode 100644 index 000000000000..231f3fcd80a7 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/core/config.hpp @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/core/config.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::internal { + + template struct banned_reinterpret_cast { + static_assert(sizeof...(arg_types) == 0, + "Sorry, but reinterpret_cast is banned in this library! Only use our public-facing include in your code! Or, if you're inside our own headers, remove " + "the std library include you just added."); + }; + + template struct banned_const_cast { + static_assert(sizeof...(arg_types) == 0, + "Sorry, but const_cast is banned in this library! Only use our public-facing include in your code! Or, if you're inside our own headers, remove " + "the std library include you just added."); + }; + + template struct banned_dynamic_cast { + static_assert(sizeof...(arg_types) == 0, + "Sorry, but dynamic_cast is banned in this library! Only use our public-facing include in your code! Or, if you're inside our own headers, remove " + "the std library include you just added."); + }; + +} + +namespace jsonifier { + + struct serialize_options { + uint64_t indentSize{ 3 }; + uint64_t indent{}; + char indentChar{ ' ' }; + bool prescanStrings{}; + bool prettify{}; + }; + + struct parse_options { + uint64_t defaultBatchSize{ 1024 * 1024 }; + uint64_t maxDepth{ 1024 }; + bool nullTerminated{ true }; + bool newLineDelimited{}; + bool partialRead{}; + bool knownOrder{}; + bool minified{}; + }; + + struct prettify_options { + uint64_t indentSize{ 3 }; + char indentChar{ ' ' }; + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/core/defines.hpp b/deps/jsonifier/include/jsonifier-incl/core/defines.hpp new file mode 100644 index 000000000000..b804f3ee2a0b --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/core/defines.hpp @@ -0,0 +1,389 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/core/defines.hpp + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(JSONIFIER_ARCH_X64) + #if defined(__x86_64__) || defined(_M_X64) + #define JSONIFIER_ARCH_X64 1 + #else + #define JSONIFIER_ARCH_X64 0 + #endif +#endif + +#if !defined(JSONIFIER_ARCH_ARM64) + #if defined(__aarch64__) || defined(_M_ARM64) + #define JSONIFIER_ARCH_ARM64 1 + #else + #define JSONIFIER_ARCH_ARM64 0 + #endif +#endif + +#if !defined(JSONIFIER_PLATFORM_WINDOWS) + #if defined(_WIN32) + #define JSONIFIER_PLATFORM_WINDOWS 1 + #else + #define JSONIFIER_PLATFORM_WINDOWS 0 + #endif +#endif + +#if !defined(JSONIFIER_PLATFORM_ANDROID) + #if defined(__ANDROID__) + #define JSONIFIER_PLATFORM_ANDROID 1 + #else + #define JSONIFIER_PLATFORM_ANDROID 0 + #endif +#endif + +#if !defined(JSONIFIER_PLATFORM_LINUX) + #if defined(__linux__) && !defined(__ANDROID__) + #define JSONIFIER_PLATFORM_LINUX 1 + #else + #define JSONIFIER_PLATFORM_LINUX 0 + #endif +#endif + +#if !defined(JSONIFIER_PLATFORM_MAC) + #if defined(__APPLE__) + #define JSONIFIER_PLATFORM_MAC 1 + #else + #define JSONIFIER_PLATFORM_MAC 0 + #endif +#endif + +#if !defined(JSONIFIER_COMPILER_CLANG) + #if defined(__clang__) + #define JSONIFIER_COMPILER_CLANG 1 + #else + #define JSONIFIER_COMPILER_CLANG 0 + #endif +#endif + +#if !defined(JSONIFIER_COMPILER_GCC) + #if defined(__GNUC__) && !JSONIFIER_COMPILER_CLANG && !defined(__INTEL_LLVM_COMPILER) && !defined(__NVCOMPILER) + #define JSONIFIER_COMPILER_GCC 1 + #else + #define JSONIFIER_COMPILER_GCC 0 + #endif +#endif + +#if !defined(JSONIFIER_COMPILER_MSVC) + #if defined(_MSC_VER) && !JSONIFIER_COMPILER_CLANG + #define JSONIFIER_COMPILER_MSVC 1 + #else + #define JSONIFIER_COMPILER_MSVC 0 + #endif +#endif + +#if !defined(JSONIFIER_LZCNT) + #if defined(__LZCNT__) || (JSONIFIER_COMPILER_MSVC && defined(__AVX2__)) + #define JSONIFIER_LZCNT 1 + #else + #define JSONIFIER_LZCNT 0 + #endif +#endif + +#if !defined(JSONIFIER_POPCNT) + #if defined(__POPCNT__) || (JSONIFIER_COMPILER_MSVC && defined(__AVX__)) + #define JSONIFIER_POPCNT 1 + #else + #define JSONIFIER_POPCNT 0 + #endif +#endif + +#if !defined(JSONIFIER_BMI) + #if defined(__BMI__) || (JSONIFIER_COMPILER_MSVC && defined(__AVX2__)) + #define JSONIFIER_BMI 1 + #else + #define JSONIFIER_BMI 0 + #endif +#endif + +#if !defined(JSONIFIER_CLMUL) + #if defined(__PCLMUL__) || defined(__ARM_FEATURE_AES) || defined(__ARM_FEATURE_CRYPTO) || (JSONIFIER_COMPILER_MSVC && (defined(__AVX2__) || JSONIFIER_ARCH_ARM64)) + #define JSONIFIER_CLMUL 1 + #else + #define JSONIFIER_CLMUL 0 + #endif +#endif + +#if !defined(JSONIFIER_AVX) + #if defined(__AVX__) + #define JSONIFIER_AVX 1 + #else + #define JSONIFIER_AVX 0 + #endif +#endif + +#if !defined(JSONIFIER_AVX2) + #if defined(__AVX2__) + #define JSONIFIER_AVX2 1 + #else + #define JSONIFIER_AVX2 0 + #endif +#endif + +#if !defined(JSONIFIER_AVX512) + #if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512VBMI2__) + #define JSONIFIER_AVX512 1 + #else + #define JSONIFIER_AVX512 0 + #endif +#endif + +#if !defined(JSONIFIER_SVE2) + #if defined(__ARM_FEATURE_SVE2) + #define JSONIFIER_SVE2 1 + #else + #define JSONIFIER_SVE2 0 + #endif +#endif + +#if !defined(JSONIFIER_NEON) + #if (defined(__ARM_NEON) || JSONIFIER_ARCH_ARM64) && !JSONIFIER_SVE2 + #define JSONIFIER_NEON 1 + #else + #define JSONIFIER_NEON 0 + #endif +#endif + +#if !defined(JSONIFIER_OPTIMIZED) + #if defined(__OPTIMIZE__) || (JSONIFIER_COMPILER_MSVC && !defined(_DEBUG)) + #define JSONIFIER_OPTIMIZED 1 + #else + #define JSONIFIER_OPTIMIZED 0 + #endif +#endif + +#if !defined(JSONIFIER_INLINE) + #if JSONIFIER_OPTIMIZED + #if JSONIFIER_COMPILER_MSVC + #define JSONIFIER_INLINE [[msvc::forceinline]] inline + #else + #define JSONIFIER_INLINE inline __attribute__((always_inline)) + #endif + #else + #define JSONIFIER_INLINE inline + #endif +#endif + +#if !defined(JSONIFIER_NOINLINE) + #if JSONIFIER_COMPILER_MSVC + #define JSONIFIER_NOINLINE __declspec(noinline) inline + #else + #define JSONIFIER_NOINLINE inline __attribute__((noinline)) + #endif +#endif + +#if !defined(JSONIFIER_EXCEPTIONS) + #if defined(__cpp_exceptions) || defined(_CPPUNWIND) + #define JSONIFIER_EXCEPTIONS 1 + #else + #define JSONIFIER_EXCEPTIONS 0 + #endif +#endif + +#if JSONIFIER_EXCEPTIONS + #define JSONIFIER_THROW(...) throw __VA_ARGS__ +#else + #define JSONIFIER_THROW(...) std::abort() +#endif + +#if !defined(JSONIFIER_NO_SANITIZE_ADDRESS) + #if JSONIFIER_COMPILER_MSVC + #define JSONIFIER_NO_SANITIZE_ADDRESS __declspec(no_sanitize_address) + #else + #define JSONIFIER_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) + #endif +#endif + +#if !defined(JSONIFIER_LIFETIME_BOUND) + #if JSONIFIER_COMPILER_CLANG + #define JSONIFIER_LIFETIME_BOUND [[clang::lifetimebound]] + #elif JSONIFIER_COMPILER_MSVC + #define JSONIFIER_LIFETIME_BOUND [[msvc::lifetimebound]] + #else + #define JSONIFIER_LIFETIME_BOUND + #endif +#endif + +#if !defined(JSONIFIER_TUPLET_NO_UNIQUE_ADDRESS) + #if JSONIFIER_COMPILER_MSVC + #define JSONIFIER_TUPLET_NO_UNIQUE_ADDRESS [[msvc::no_unique_address]] + #else + #define JSONIFIER_TUPLET_NO_UNIQUE_ADDRESS [[no_unique_address]] + #endif +#endif + +#if !defined(JSONIFIER_INLINE_EXCEPT_MAC_GCC) + #if JSONIFIER_OPTIMIZED + #if !(JSONIFIER_COMPILER_MSVC || (JSONIFIER_PLATFORM_MAC && JSONIFIER_COMPILER_GCC)) + #define JSONIFIER_INLINE_EXCEPT_MAC_GCC inline __attribute__((always_inline)) + #else + #define JSONIFIER_INLINE_EXCEPT_MAC_GCC inline + #endif + #else + #define JSONIFIER_INLINE_EXCEPT_MAC_GCC inline + #endif +#endif + +#if !defined(JSONIFIER_INLINE_LINUX_GCC) + #if JSONIFIER_OPTIMIZED && JSONIFIER_PLATFORM_LINUX && JSONIFIER_COMPILER_GCC + #define JSONIFIER_INLINE_LINUX_GCC inline __attribute__((always_inline)) + #else + #define JSONIFIER_INLINE_LINUX_GCC inline + #endif +#endif + +#if JSONIFIER_PLATFORM_WINDOWS && !defined(NOMINMAX) + #define NOMINMAX +#endif + +#if JSONIFIER_PLATFORM_WINDOWS && !defined(WIN32_LEAN_AND_MEAN) + #define WIN32_LEAN_AND_MEAN +#endif + +#if !JSONIFIER_COMPILER_MSVC && !JSONIFIER_COMPILER_GCC && !JSONIFIER_COMPILER_CLANG + #error "Sorry, this compiler is not supported." +#endif + +#if !defined(JSONIFIER_HAS_INT128) + #if defined(__SIZEOF_INT128__) + #define JSONIFIER_HAS_INT128 1 + #else + #define JSONIFIER_HAS_INT128 0 + #endif +#endif + +#if (JSONIFIER_AVX || JSONIFIER_AVX2 || JSONIFIER_AVX512) && !JSONIFIER_ARCH_X64 + #error "The AVX backends require an x86-64 target." +#endif + +#if (JSONIFIER_NEON || JSONIFIER_SVE2) && !JSONIFIER_ARCH_ARM64 + #error "The NEON and SVE2 backends require an arm64 target." +#endif + +#if JSONIFIER_ANY_AVX || (JSONIFIER_ARCH_X64 && (JSONIFIER_LZCNT || JSONIFIER_POPCNT || JSONIFIER_BMI)) + #include +#elif JSONIFIER_NEON + #include +#elif JSONIFIER_SVE2 + #include + #include + #include +#endif + +#if JSONIFIER_COMPILER_MSVC + #include +#endif + +#if JSONIFIER_PLATFORM_WINDOWS + #include +#else + #include + #include + #include + #include + #include +#endif + +#if defined(__SANITIZE_ADDRESS__) + #define JSONIFIER_ASAN_ENABLED 1 +#elif defined(__has_feature) + #if __has_feature(address_sanitizer) + #define JSONIFIER_ASAN_ENABLED 1 + #endif +#endif +#if !defined(JSONIFIER_ASAN_ENABLED) + #define JSONIFIER_ASAN_ENABLED 0 +#endif + +#if JSONIFIER_COMPILER_GCC || JSONIFIER_COMPILER_CLANG + #define JSONIFIER_ASM(x) asm x +#else + #define JSONIFIER_ASM(x) +#endif + +#ifdef JSONIFIER_USE_INT128 +#elif defined(__SIZEOF_INT128__) + #define JSONIFIER_USE_INT128 1 +#else + #define JSONIFIER_USE_INT128 0 +#endif + +template inline void jsonifierFailMemcpyImpl(arg_types&&...) noexcept { + static_assert(sizeof...(arg_types) == 0, + "Sorry, but un-constrained memcpy is banned in this library! Only use our public-facing include in your code! Or, if you're inside our own headers, remove " + "the std library include you just added."); +} + +template inline void jsonifierFailMemmoveImpl(arg_types&&...) noexcept { + static_assert(sizeof...(arg_types) == 0, + "Sorry, but un-constrained memmove is banned in this library! Only use our public-facing include in your code! Or, if you're inside our own headers, remove " + "the std library include you just added."); +} + +namespace std { + + template inline void jsonifierFailMemmoveImpl(arg_types&&... args) noexcept { + ::jsonifierFailMemmoveImpl(args...); + } + + template inline void jsonifierFailMemcpyImpl(arg_types&&... args) noexcept { + ::jsonifierFailMemcpyImpl(args...); + } + +} + +namespace jsonifier::internal { + + template JSONIFIER_INLINE void pow2MemmoveWrapper(value_type_01* dst, const value_type_02* src) noexcept { + static_assert(std::has_single_bit(size), "Sorry, but you can only memmove a power-of-2 size."); + alignas(size) uint8_t scratch[size]; + std::memcpy(scratch, src, size); + std::memcpy(dst, scratch, size); + } + + template JSONIFIER_INLINE void memmoveWrapper(value_type_01* dst, const value_type_02* src, uint64_t size) noexcept { + std::memmove(dst, src, size); + } + + template + JSONIFIER_INLINE void memcpyWrapper(value_type_01* __restrict dst, const value_type_02* __restrict src, uint64_t size) noexcept { + std::memcpy(dst, src, size); + } + + template + JSONIFIER_INLINE void pow2MemcpyWrapper(value_type_01* __restrict dst, const value_type_02* __restrict src) noexcept { + static_assert(std::has_single_bit(size), "Sorry, but you can only memcpy a power-of-2 size."); + std::memcpy(dst, src, size); + } + +} + +#define reinterpret_cast jsonifier::internal::banned_reinterpret_cast +#define const_cast jsonifier::internal::banned_const_cast +#define dynamic_cast jsonifier::internal::banned_dynamic_cast +#define memcpy(...) internal::jsonifierFailMemcpyImpl(__VA_ARGS__) +#define memmove(...) internal::jsonifierFailMemmoveImpl(__VA_ARGS__) diff --git a/deps/jsonifier/include/jsonifier-incl/core/fastio.hpp b/deps/jsonifier/include/jsonifier-incl/core/fastio.hpp new file mode 100644 index 000000000000..21ac3d4abaa5 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/core/fastio.hpp @@ -0,0 +1,244 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/core/fastio.hpp + */ +#pragma once + +#include + +namespace jsonifier::internal { + + enum class stream_targets : uint8_t { stdout_target, stderr_target }; + + inline uint64_t rawWrite(stream_targets stream_target, const char* data, uint64_t len) noexcept { + if (len == 0 || data == nullptr) { + return 0; + } +#if JSONIFIER_PLATFORM_WINDOWS + static constexpr DWORD handle_ids[2]{ STD_OUTPUT_HANDLE, STD_ERROR_HANDLE }; + static constexpr uint64_t max_chunk = 1ull << 30; + HANDLE handle = GetStdHandle(handle_ids[static_cast(stream_target)]); + if (handle == nullptr || handle == INVALID_HANDLE_VALUE) { + return 0; + } + uint64_t total = 0; + while (total < len) { + uint64_t remaining = len - total; + DWORD chunk = static_cast(remaining < max_chunk ? remaining : max_chunk); + DWORD written = 0; + if (!WriteFile(handle, data + total, chunk, &written, nullptr) || written == 0) { + break; + } + total += written; + } + return total; +#else + static constexpr int32_t stream_types[2]{ 1, 2 }; + int32_t fd = stream_types[static_cast(stream_target)]; + uint64_t total = 0; + while (total < len) { + ssize_t result = write(fd, data + total, len - total); + if (result < 0) { + if (errno == EINTR) { + continue; + } + if (errno == EAGAIN || errno == EWOULDBLOCK) { + pollfd pfd{ fd, POLLOUT, 0 }; + if (poll(&pfd, 1, -1) < 0 && errno != EINTR) { + break; + } + continue; + } + break; + } + if (result == 0) { + break; + } + total += static_cast(result); + } + return total; +#endif + } + + alignas(64) inline constexpr uint8_t digitCounts[]{ 19, 19, 19, 19, 18, 18, 18, 17, 17, 17, 16, 16, 16, 16, 15, 15, 15, 14, 14, 14, 13, 13, 13, 13, 12, 12, 12, 11, 11, 11, 10, + 10, 10, 10, 9, 9, 9, 8, 8, 8, 7, 7, 7, 7, 6, 6, 6, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3, 2, 2, 2, 1, 1, 1, 1, 1 }; + + alignas(64) inline constexpr uint64_t digitCountThresholds[]{ 0ULL, 9ULL, 99ULL, 999ULL, 9999ULL, 99999ULL, 999999ULL, 9999999ULL, 99999999ULL, 999999999ULL, 9999999999ULL, + 99999999999ULL, 999999999999ULL, 9999999999999ULL, 99999999999999ULL, 999999999999999ULL, 9999999999999999ULL, 99999999999999999ULL, 999999999999999999ULL, + 9999999999999999999ULL }; + + inline uint64_t fastDigitCount(const uint64_t inputValue) noexcept { + const uint64_t originalDigitCount{ digitCounts[simd::countlZero(inputValue)] }; + return originalDigitCount + static_cast(inputValue > digitCountThresholds[originalDigitCount]); + } + + template inline uint64_t writeFloat(char* dest, value_type value) noexcept { + auto result = std::to_chars(dest, dest + 64, value); + return static_cast(result.ptr - dest); + } + + inline char* writeUnsigned(char* bufferEnd, uint64_t value) noexcept { + char* pos = bufferEnd; + alignas(64) static constexpr char digitPairs[201] = "0001020304050607080910111213141516171819" + "2021222324252627282930313233343536373839" + "4041424344454647484950515253545556575859" + "6061626364656667686970717273747576777879" + "8081828384858687888990919293949596979899"; + while (value >= 100) { + auto pair = value % 100; + value /= 100; + pos -= 2; + pos[0] = digitPairs[pair * 2]; + pos[1] = digitPairs[pair * 2 + 1]; + } + if (value < 10) { + *--pos = static_cast('0' + value); + } else { + pos -= 2; + pos[0] = digitPairs[value * 2]; + pos[1] = digitPairs[value * 2 + 1]; + } + return pos; + } + + template inline uint64_t writeInteger(char* dest, value_type value) noexcept { + if constexpr (std::is_signed_v) { + uint64_t magnitude; + char* outLocal = dest; + if (value < 0) { + *outLocal++ = '-'; + magnitude = ~static_cast(value) + 1; + } else { + magnitude = static_cast(value); + } + auto count = fastDigitCount(magnitude); + char* end = outLocal + count; + writeUnsigned(end, magnitude); + return static_cast(end - dest); + } else { + uint64_t magnitude = static_cast(value); + auto count = fastDigitCount(magnitude); + char* end = dest + count; + writeUnsigned(end, magnitude); + return count; + } + } + + struct endl_t {}; + inline constexpr endl_t endl{}; + + struct flush_t {}; + inline constexpr flush_t flush{}; + + template struct basic_stream { + inline explicit basic_stream(stream_targets streamTargetNew) noexcept : bufferLength{ 0 }, stream_target{ streamTargetNew } { + } + + basic_stream& operator=(const basic_stream&) = delete; + basic_stream(const basic_stream&) = delete; + + template + inline std::enable_if_t && !std::is_same_v && !std::is_same_v, basic_stream&> operator<<( + integer_type value) noexcept { + ensureSpace(21); + bufferLength += writeInteger(streamBuffer + bufferLength, value); + return *this; + } + + template inline std::enable_if_t, basic_stream&> operator<<(float_type value) noexcept { + ensureSpace(64); + bufferLength += writeFloat(streamBuffer + bufferLength, value); + return *this; + } + + inline basic_stream& operator<<(std::string_view value) noexcept { + writeRaw(value.data(), value.size()); + return *this; + } + + inline void writeRaw(const char* data, uint64_t size) noexcept { + if (size >= bufferSize) { + doFlush(); + rawWrite(stream_target, data, size); + return; + } + ensureSpace(size); + jsonifierMemcpy(streamBuffer + bufferLength, data, size); + bufferLength += size; + } + + inline basic_stream& operator<<(bool value) noexcept { + return (*this) << std::string_view(value ? "true" : "false"); + } + + inline basic_stream& operator<<(const char* value) noexcept { + return (*this) << std::string_view(value); + } + + inline basic_stream& operator<<(char value) noexcept { + ensureSpace(1); + streamBuffer[bufferLength++] = value; + return *this; + } + + inline void doFlush() noexcept { + if (bufferLength > 0) { + rawWrite(stream_target, streamBuffer, bufferLength); + bufferLength = 0; + } + } + + inline void ensureSpace(uint64_t needed) noexcept { + if (bufferLength + needed > bufferSize) { + doFlush(); + } + } + + inline basic_stream& operator<<(flush_t) noexcept { + doFlush(); + return *this; + } + + inline basic_stream& operator<<(endl_t) noexcept { + (*this) << '\n'; + return *this; + } + + inline std::string_view view() const noexcept { + return { streamBuffer, static_cast(bufferLength) }; + } + + inline void flushNow() noexcept { + doFlush(); + } + + inline void discard() noexcept { + bufferLength = 0; + } + + inline ~basic_stream() noexcept { + doFlush(); + } + + uint64_t bufferLength; + stream_targets stream_target; + char streamBuffer[bufferSize]; + }; + +#if JSONIFIER_COMPILER_CLANG + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wexit-time-destructors" + #pragma clang diagnostic ignored "-Wglobal-constructors" +#endif + + inline basic_stream<> cout{ stream_targets::stdout_target }; + + inline basic_stream<> err{ stream_targets::stderr_target }; + +#if JSONIFIER_COMPILER_CLANG + #pragma clang diagnostic pop +#endif + +} diff --git a/deps/jsonifier/include/jsonifier-incl/core/jsonifier_core.hpp b/deps/jsonifier/include/jsonifier-incl/core/jsonifier_core.hpp new file mode 100644 index 000000000000..b553d56db218 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/core/jsonifier_core.hpp @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/core/jsonifier_core.hpp + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace jsonifier { + + class jsonifier_core : public internal::generic_iterator { + public: + friend struct internal::generic_iterator; + + inline jsonifier_core() noexcept = default; + + inline jsonifier_core& operator=(jsonifier_core&& other) noexcept { + if (this != &other) [[likely]] { + podSection = internal::move(other.podSection); + section = internal::move(other.section); + genericData = internal::move(other.genericData); + } + return *this; + } + + inline jsonifier_core(jsonifier_core&& other) noexcept + : podSection{ internal::move(other.podSection) }, section{ internal::move(other.section) }, genericData{ internal::move(other.genericData) } { + } + + jsonifier_core& operator=(const jsonifier_core& other) = delete; + jsonifier_core(const jsonifier_core& other) = delete; + + template + inline generic::document_stream iterateMany(const buffer_type& in, uint64_t batchSize = options.defaultBatchSize) noexcept { + return generic::document_stream{ *this, std::bit_cast(in.data()), in.size(), batchSize }; + } + + inline ~jsonifier_core() noexcept = default; + + protected: + template friend class generic::document_stream; + + JSONIFIER_INLINE internal::generic_storage& genericStorage() noexcept { + return genericData; + } + + internal::pod_simd_string_reader podSection{}; + internal::simd_string_reader section{}; + internal::generic_storage genericData{}; + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/index.hpp b/deps/jsonifier/include/jsonifier-incl/index.hpp new file mode 100644 index 000000000000..6a4e998a3df5 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/index.hpp @@ -0,0 +1,9 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/index.hpp + */ +#pragma once + +#include diff --git a/deps/jsonifier/include/jsonifier-incl/parsing/document_state.hpp b/deps/jsonifier/include/jsonifier-incl/parsing/document_state.hpp new file mode 100644 index 000000000000..68bef23c58c8 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/parsing/document_state.hpp @@ -0,0 +1,182 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/parsing/document_state.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::internal { + + struct document_state { + static constexpr uint8_t reverseEngageScore{ 4 }; + static constexpr uint8_t reverseMaxScore{ 8 }; + + JSONIFIER_INLINE generic::error_codes advanceChild(read_structural_index_ptr childValue, uint64_t containerDepth, uint8_t closer, + read_structural_index_ptr& next) noexcept { + const read_buffer_ptr base = root; + read_structural_index_ptr limit = tapeEnd; + read_structural_index_ptr position = cursor; + uint64_t currentDepth = depth; + if (currentDepth == containerDepth) { + if (position <= childValue) { + position = skipValue(base, childValue, limit); + } + } else if (currentDepth > containerDepth) { + position = finishToDepth(base, position, limit, currentDepth, containerDepth); + } + generic::error_codes error_code{ generic::error_codes::tape_error }; + if (position >= limit) [[unlikely]] { + error_code = generic::error_codes::unclosed_container; + } else if (const uint8_t separator = base[*position]; separator == uint_c_v<','>) [[likely]] { + ++position; + next = position; + error_code = position < limit ? generic::error_codes::success : generic::error_codes::unclosed_container; + } else if (separator == closer) [[likely]] { + ++position; + --currentDepth; + next = nullptr; + error_code = generic::error_codes::success; + } + cursor = position; + depth = currentDepth; + return error_code; + } + + JSONIFIER_INLINE static unrestrict_t finishToDepth(read_buffer_ptr base, read_structural_index_ptr position, read_structural_index_ptr limit, + uint64_t& currentDepth, uint64_t targetDepth) noexcept { + int64_t localDepth{ static_cast(currentDepth) }; + const int64_t target{ static_cast(targetDepth) }; +#if JSONIFIER_PLATFORM_LINUX && JSONIFIER_COMPILER_GCC + while (localDepth > target && position < limit) { + localDepth += depthDeltaTable[base[*position]]; + ++position; + } +#else + if (localDepth > target) { + while (position < limit) { + localDepth += depthDeltaTable[base[*position]]; + ++position; + if (localDepth <= target) { + break; + } + } + } +#endif + currentDepth = static_cast(localDepth); + return position; + } + + template JSONIFIER_INLINE bool keyMatches(read_structural_index_ptr iter, string_view key) const noexcept { + const uint64_t keySize = key.size(); + const uint64_t span = static_cast(*(iter + 1) - *iter); + read_buffer_ptr keyStart = root + *iter + 1; + if (span == keySize + 2) [[likely]] { + return internal::comparison::compare(keyStart, key.data(), keySize); + } + if constexpr (minified) { + return false; + } else { + return span > keySize + 2 && keyStart[keySize] == uint_c_v<'"'> && internal::comparison::compare(keyStart, key.data(), keySize); + } + } + + JSONIFIER_INLINE static unrestrict_t skipValue(read_buffer_ptr base, read_structural_index_ptr iter, read_structural_index_ptr limit) noexcept { + if (!isOpener(base[*iter])) [[likely]] { + return iter + 1; + } + int64_t openCount{ 1 }; + ++iter; + while (iter < limit) { + openCount += depthDeltaTable[base[*iter]]; + ++iter; + if (openCount == 0) { + return iter; + } + } + return limit; + } + + JSONIFIER_INLINE bool keyHasEscape(read_structural_index_ptr iter) const noexcept { + read_buffer_ptr keyStart = root + *iter + 1; + read_buffer_ptr keyLimit = root + *(iter + 1); + return keyLimit > keyStart && char_comparison<'\\', uint8_t>::memchar(keyStart, static_cast(keyLimit - keyStart)); + } + + JSONIFIER_INLINE unrestrict_t skipValue(read_structural_index_ptr iter) const noexcept { + return skipValue(root, iter, tapeEnd); + } + + template JSONIFIER_INLINE string_view rawKey(read_structural_index_ptr iter) const noexcept { + read_buffer_ptr keyStart = root + *iter + 1; + read_buffer_ptr keyEnd = root + *(iter + 1) - 1; + if constexpr (!minified) { + while (keyEnd > keyStart && *keyEnd != uint_c_v<'"'>) { + --keyEnd; + } + } + return string_view{ std::bit_cast(keyStart), static_cast(keyEnd - keyStart) }; + } + + JSONIFIER_INLINE bool unescapedKeyMatches(read_structural_index_ptr iter, string_view key) const noexcept { + read_buffer_ptr keyStart = root + *iter + 1; + char* unescaped = arena->scratch(); + const auto scanned = string_parser::impl(keyStart, end, unescaped); + return scanned.outLength == key.size() && internal::comparison::compare(unescaped, key.data(), key.size()); + } + + template JSONIFIER_INLINE static word_type loadWord(read_buffer_ptr source) noexcept { + word_type word; + pow2MemcpyWrapper(&word, source); + return word; + } + + template JSONIFIER_INLINE bool charAt(read_structural_index_ptr iter) const noexcept { + return root[*iter] == value; + } + + JSONIFIER_INLINE uint8_t charAt(read_structural_index_ptr iter) const noexcept { + return root[*iter]; + } + + JSONIFIER_INLINE void finishToDepth(uint64_t targetDepth) noexcept { + uint64_t currentDepth{ depth }; + cursor = finishToDepth(root, cursor, tapeEnd, currentDepth, targetDepth); + depth = currentDepth; + } + + JSONIFIER_INLINE void noteBackwardHit() noexcept { + reverseScore += static_cast(reverseScore < reverseMaxScore); + } + + JSONIFIER_INLINE static bool isOpener(uint8_t c) noexcept { + return (c | 0x20) == uint_c_v<'{'>; + } + + JSONIFIER_INLINE static bool isCloser(uint8_t c) noexcept { + return (c | 0x20) == uint_c_v<'}'>; + } + + JSONIFIER_INLINE void noteForwardHit() noexcept { + reverseScore -= static_cast(reverseScore > 0); + } + + JSONIFIER_INLINE bool reverseMode() const noexcept { + return reverseScore >= reverseEngageScore; + } + + read_structural_index_ptr tapeEnd{}; + read_structural_index_ptr cursor{}; + read_structural_index_ptr tape{}; + field_index_map* fieldIndex{}; + read_buffer_ptr root{}; + read_buffer_ptr end{}; + string_arena* arena{}; + uint64_t depth{}; + uint8_t reverseScore{}; + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/parsing/document_stream.hpp b/deps/jsonifier/include/jsonifier-incl/parsing/document_stream.hpp new file mode 100644 index 000000000000..99c7641836fb --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/parsing/document_stream.hpp @@ -0,0 +1,350 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/parsing/document_stream.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::generic { + + template class document_stream { + public: + using document_state = internal::document_state; + using document = generic_document; + + class iterator { + public: + using iterator_category = std::input_iterator_tag; + using value_type = document; + using difference_type = std::ptrdiff_t; + + JSONIFIER_INLINE iterator() noexcept = default; + + JSONIFIER_INLINE explicit iterator(document_stream* streamNew) noexcept : stream{ streamNew } { + } + + JSONIFIER_INLINE bool operator==(const iterator& other) const noexcept { + return exhausted() == other.exhausted(); + } + + JSONIFIER_INLINE uint64_t currentIndex() const noexcept { + return stream->currentIndex(); + } + + JSONIFIER_INLINE string_view source() const noexcept { + return stream->source(); + } + + JSONIFIER_INLINE document operator*() const noexcept { + return stream->current(); + } + + JSONIFIER_INLINE void operator++(int32_t) noexcept { + stream->advance(); + } + + JSONIFIER_INLINE iterator& operator++() noexcept { + stream->advance(); + return *this; + } + + protected: + JSONIFIER_INLINE bool exhausted() const noexcept { + return !stream || stream->finished; + } + + document_stream* stream{}; + }; + + JSONIFIER_INLINE document_stream(parser_type& parserNew, internal::read_buffer_ptr inputNew, uint64_t inputLengthNew, uint64_t batchSizeNew) noexcept + : input{ inputNew }, inputLength{ inputNew ? inputLengthNew : 0 }, parser{ &parserNew }, + batchSize{ std::clamp(batchSizeNew, minimumBatchSize, maximumBatchSize) } { + } + + document_stream& operator=(const document_stream&) = delete; + document_stream(const document_stream&) = delete; + + JSONIFIER_INLINE uint64_t truncatedBytes() const noexcept { + return truncated; + } + + JSONIFIER_INLINE error_codes error() const noexcept { + return error_code; + } + + JSONIFIER_INLINE iterator end() const noexcept { + return iterator{}; + } + + JSONIFIER_INLINE iterator begin() noexcept { + if (!started) { + started = true; + if (inputLength == 0) { + finished = true; + } else { + loadWindow(0); + locate(windowTape); + } + } + return iterator{ this }; + } + + protected: + static constexpr uint64_t minimumBatchSize{ 64 }; + static constexpr uint64_t maximumBatchSize{ std::numeric_limits::max() - 64 }; + + JSONIFIER_INLINE internal::unrestrict_t findDocumentEnd(internal::read_structural_index_ptr start) noexcept { + document_state& state = docState(); + const uint8_t first = state.charAt(start); + if (document_state::isOpener(first)) { + int64_t openCount{ 1 }; + for (internal::read_structural_index_ptr iter = start + 1; iter < windowTapeEnd; ++iter) { + openCount += internal::depthDeltaTable[state.charAt(iter)]; + if (openCount == 0) { + return iter + 1; + } + } + return nullptr; + } + if (document_state::isCloser(first) || first == internal::uint_c_v<','> || first == internal::uint_c_v<':'>) [[unlikely]] { + error_code = error_codes::tape_error; + return nullptr; + } + if (start + 1 == windowTapeEnd && !finalWindow) { + return nullptr; + } + return start + 1; + } + + JSONIFIER_INLINE void bindDocument(internal::read_structural_index_ptr begin, internal::read_structural_index_ptr end) noexcept { + docBegin = begin; + docEnd = end; + pendingSeparator = false; + document_state& state = docState(); + state.tape = begin; + state.tapeEnd = end; + state.cursor = begin; + state.depth = 0; + storage().fieldIndex.reset(); + } + + JSONIFIER_INLINE internal::unrestrict_t lastDocumentStart() const noexcept { + document_state& state = docState(); + for (internal::read_structural_index_ptr iter = windowTapeEnd - 1; iter > windowTape; --iter) { + if (startsDocument(state.charAt(iter)) && endsValue(state.charAt(iter - 1))) { + return iter; + } + } + return windowTape; + } + + JSONIFIER_INLINE static bool startsDocument(uint8_t c) noexcept { + return !document_state::isCloser(c) && c != internal::uint_c_v<','> && c != internal::uint_c_v<':'>; + } + + JSONIFIER_INLINE static bool endsValue(uint8_t c) noexcept { + return !document_state::isOpener(c) && c != internal::uint_c_v<','> && c != internal::uint_c_v<':'>; + } + + JSONIFIER_INLINE void locateDelimited(internal::read_structural_index_ptr next) noexcept { + while (true) { + if (next == windowTapeEnd) { + if (finalWindow) { + if (pendingSeparator) [[unlikely]] { + error_code = error_codes::trailing_content; + } else { + finished = true; + } + return; + } + loadWindow(windowStart + windowLength); + next = windowTape; + continue; + } + if (docBegin && !pendingSeparator && docState().template charAt>(next)) { + ++next; + pendingSeparator = true; + continue; + } + if (internal::read_structural_index_ptr documentEnd = findDocumentEnd(next); documentEnd) [[likely]] { + bindDocument(next, documentEnd); + return; + } + if (error_code != error_codes::success) [[unlikely]] { + return; + } + if (finalWindow) { + truncated = windowLength - *next; + finished = true; + return; + } + if (next == windowTape) [[unlikely]] { + error_code = error_codes::capacity; + return; + } + loadWindow(windowStart + *next); + next = windowTape; + } + } + + JSONIFIER_INLINE internal::unrestrict_t consumedEnd() const noexcept { + document_state& state = docState(); + if (state.depth > 0) { + state.finishToDepth(0); + return state.cursor; + } + if (state.cursor > docBegin) { + return state.cursor; + } + return state.skipValue(docBegin); + } + + JSONIFIER_INLINE void locateWhitespace(internal::read_structural_index_ptr next) noexcept { + while (true) { + if (next == completeEnd) { + if (finalWindow) { + finished = true; + return; + } + if (completeEnd == windowTape) [[unlikely]] { + error_code = error_codes::capacity; + return; + } + loadWindow(windowStart + (completeEnd == windowTapeEnd ? windowLength : *completeEnd)); + next = windowTape; + continue; + } + if (!startsDocument(docState().charAt(next))) [[unlikely]] { + error_code = error_codes::tape_error; + return; + } + bindDocument(next, completeEnd); + return; + } + } + + JSONIFIER_INLINE void findCompleteEnd() noexcept { + if (windowTape == windowTapeEnd) { + completeEnd = windowTapeEnd; + return; + } + internal::read_structural_index_ptr lastStart = lastDocumentStart(); + if (lastStart != windowTape && !finalWindow) { + completeEnd = lastStart; + return; + } + if (!startsDocument(docState().charAt(lastStart)) || findDocumentEnd(lastStart)) { + completeEnd = windowTapeEnd; + return; + } + completeEnd = lastStart; + if (finalWindow) { + truncated = windowLength - *lastStart; + } + } + + JSONIFIER_INLINE string_view source() const noexcept { + if (error_code != error_codes::success) [[unlikely]] { + return {}; + } + internal::read_buffer_ptr base = input + windowStart; + const uint64_t start = *docBegin; + uint64_t stop{}; + if constexpr (!options.newLineDelimited) { + stop = *docEnd; + } else { + stop = *docState().skipValue(docBegin); + } + while (stop > start && isSeparatorByte(base[stop - 1])) { + --stop; + } + return string_view{ std::bit_cast(base + start), stop - start }; + } + + JSONIFIER_INLINE uint64_t currentIndex() const noexcept { + return error_code == error_codes::success ? windowStart + *docBegin : windowStart; + } + + JSONIFIER_INLINE void locate(internal::read_structural_index_ptr next) noexcept { + if constexpr (!options.newLineDelimited) { + locateDelimited(next); + } else { + locateWhitespace(next); + } + } + + JSONIFIER_INLINE void loadWindow(uint64_t start) noexcept { + windowStart = start; + windowLength = std::min(batchSize, inputLength - start); + finalWindow = start + windowLength == inputLength; + internal::read_buffer_ptr windowRoot = input + start; + parser->template indexGenericWindow(windowRoot, windowLength); + internal::generic_storage& data = storage(); + document_state& state = data.stateFor(); + windowTape = state.tape; + windowTapeEnd = state.tapeEnd; + state.root = windowRoot; + state.end = windowRoot + windowLength; + data.arena.reset(windowLength); + state.arena = &data.arena; + state.fieldIndex = &data.fieldIndex; + if constexpr (options.newLineDelimited) { + findCompleteEnd(); + } + } + + JSONIFIER_INLINE static bool isSeparatorByte(uint8_t c) noexcept { + return internal::seperatorByteTable[c]; + } + + JSONIFIER_INLINE internal::generic_storage& storage() const noexcept { + return parser->genericStorage(); + } + + JSONIFIER_INLINE document_state& docState() const noexcept { + return storage().stateFor(); + } + + JSONIFIER_INLINE document current() const noexcept { + if (error_code != error_codes::success) [[unlikely]] { + return document{ error_code }; + } + return document{ &docState(), docBegin, 0 }; + } + + JSONIFIER_INLINE void advance() noexcept { + if (error_code != error_codes::success) [[unlikely]] { + finished = true; + return; + } + if constexpr (!options.newLineDelimited) { + locate(docEnd); + } else { + locate(consumedEnd()); + } + } + + internal::read_structural_index_ptr windowTapeEnd{}; + internal::read_structural_index_ptr completeEnd{}; + internal::read_structural_index_ptr windowTape{}; + internal::read_structural_index_ptr docBegin{}; + internal::read_structural_index_ptr docEnd{}; + internal::read_buffer_ptr input{}; + uint64_t windowLength{}; + uint64_t inputLength{}; + uint64_t windowStart{}; + parser_type* parser{}; + uint64_t batchSize{}; + uint64_t truncated{}; + error_codes error_code{ error_codes::success }; + bool pendingSeparator{}; + bool finalWindow{}; + bool finished{}; + bool started{}; + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/parsing/field_index_map.hpp b/deps/jsonifier/include/jsonifier-incl/parsing/field_index_map.hpp new file mode 100644 index 000000000000..92ee833da078 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/parsing/field_index_map.hpp @@ -0,0 +1,254 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/parsing/field_index_map.hpp + */ +#pragma once + +#include + +namespace jsonifier::internal { + + struct field_index_group { +#if JSONIFIER_NEON || JSONIFIER_SVE2 + static constexpr uint64_t laneShift{ 2 }; + static constexpr uint64_t laneMask{ 0x8888888888888888ull }; +#else + static constexpr uint64_t laneShift{ 0 }; + static constexpr uint64_t laneMask{ 0xFFFFFFFFFFFFFFFFull }; +#endif + + JSONIFIER_INLINE static uint64_t match(const int8_t* controls, int8_t control) noexcept { + const jsonifier_simd_int_128 group = simd::gatherValuesU(controls); + const jsonifier_simd_int_128 needles = simd::gatherValue(static_cast(control)); + return static_cast(simd::opBitMaskRaw(simd::opCmpEqRaw(group, needles))) & laneMask; + } + + JSONIFIER_INLINE static uint64_t firstLane(uint64_t mask) noexcept { + return static_cast(simd::countrZeroUnsafe(mask)) >> laneShift; + } + + JSONIFIER_INLINE static uint64_t clearLane(uint64_t mask) noexcept { + return simd::blsr(mask); + } + }; + + class field_index_map { + public: + static constexpr uint32_t npos{ std::numeric_limits::max() }; + + JSONIFIER_INLINE field_index_map() noexcept = default; + + JSONIFIER_INLINE field_index_map& operator=(field_index_map&& other) noexcept { + if (this != &other) [[likely]] { + controls = internal::move(other.controls); + entries = internal::move(other.entries); + slotCount = std::exchange(other.slotCount, uint32_t{}); + groupMask = std::exchange(other.groupMask, uint64_t{}); + count = std::exchange(other.count, uint64_t{}); + } + return *this; + } + + JSONIFIER_INLINE field_index_map(field_index_map&& other) noexcept + : controls{ internal::move(other.controls) }, entries{ internal::move(other.entries) }, groupMask{ std::exchange(other.groupMask, uint64_t{}) }, + count{ std::exchange(other.count, uint64_t{}) }, slotCount{ std::exchange(other.slotCount, uint32_t{}) } { + } + + field_index_map& operator=(const field_index_map&) = delete; + field_index_map(const field_index_map&) = delete; + + JSONIFIER_INLINE static uint64_t hashKey(read_buffer_ptr keyData, uint64_t keyLength, uint32_t objectIndex) noexcept { + uint64_t low{}; + uint64_t high{}; + if (keyLength > 16) { + return mix64(longKeyHash(keyData, keyLength) ^ (static_cast(objectIndex) << 32), mixMultiplier); + } + if (keyLength >= 8) { + low = loadWord(keyData); + high = loadWord(keyData + keyLength - 8); + } else if (keyLength >= 4) { + low = loadWord(keyData); + high = loadWord(keyData + keyLength - 4); + } else if (keyLength > 0) { + low = static_cast(static_cast(keyData[0])) | (static_cast(static_cast(keyData[keyLength >> 1])) << 8) | + (static_cast(static_cast(keyData[keyLength - 1])) << 16); + } + const uint64_t lengthAndObject = (static_cast(objectIndex) << 32) | keyLength; + return mix64(low ^ hashSeed ^ lengthAndObject, mixMultiplier) ^ mix64(high + mixMultiplier, hashSeed ^ lengthAndObject); + } + + template + JSONIFIER_INLINE uint32_t find(uint64_t hash, uint32_t objectIndex, state_type&& state, key_type&& key) const noexcept { + const int8_t control = static_cast(hash & 0x7F); + uint64_t group = (hash >> 7) & groupMask; + for (uint64_t probe = 0; probe <= groupMask; ++probe) { + const int8_t* groupControls = controls.data() + group * groupWidth; + for (uint64_t mask = field_index_group::match(groupControls, control); mask != 0;) { + const uint64_t lane = field_index_group::firstLane(mask); + const entry& candidate = entries.data()[group * groupWidth + lane]; + if (candidate.hash == hash && candidate.object == objectIndex && matcher_type::template impl(state, key, candidate.key)) { + return candidate.key; + } + mask = field_index_group::clearLane(mask); + } + if (field_index_group::match(groupControls, emptyControl) != 0) { + return npos; + } + group = (group + 1) & groupMask; + } + return npos; + } + + JSONIFIER_INLINE void insert(uint64_t hash, uint32_t objectIndex, uint32_t keyIndex) noexcept { + if ((count + 1) * 8 > uint64_t{ slotCount } * 7) [[unlikely]] { + grow(); + } + place(entry{ hash, objectIndex, keyIndex }); + ++count; + } + + JSONIFIER_INLINE void reset() noexcept { + if (count != 0) { + std::memset(controls.data(), static_cast(emptyControl), slotCount); + count = 0; + } + } + + JSONIFIER_INLINE uint64_t size() const noexcept { + return count; + } + + protected: + struct entry { + uint64_t hash; + uint32_t object; + uint32_t key; + }; + + static constexpr uint64_t groupWidth{ 16 }; + static constexpr uint64_t initialSlots{ 128 }; + static constexpr int8_t emptyControl{ static_cast(-128) }; + static constexpr uint64_t hashSeed{ 14695981039346656037ull }; + static constexpr uint64_t mixMultiplier{ 0x9DDFEA08EB382D69ull }; + + JSONIFIER_INLINE void place(const entry& newEntry) noexcept { + uint64_t group = (newEntry.hash >> 7) & groupMask; + while (true) { + int8_t* groupControls = controls.data() + group * groupWidth; + if (const uint64_t empties = field_index_group::match(groupControls, emptyControl); empties != 0) [[likely]] { + const uint64_t lane = field_index_group::firstLane(empties); + groupControls[lane] = static_cast(newEntry.hash & 0x7F); + entries.data()[group * groupWidth + lane] = newEntry; + return; + } + group = (group + 1) & groupMask; + } + } + + template JSONIFIER_INLINE static uint64_t loadWord(read_buffer_ptr source) noexcept { + word_type word; + pow2MemcpyWrapper(&word, source); + return static_cast(word); + } + + JSONIFIER_INLINE static uint64_t longKeyHash(read_buffer_ptr keyData, uint64_t keyLength) noexcept { + static constexpr uint64_t c1{ 0x87c37b91114253d5ull }; + static constexpr uint64_t c2{ 0x4cf5ad432745937full }; + uint64_t h1 = hashSeed; + uint64_t h2 = hashSeed; + const uint64_t nblocks = keyLength / 16; + for (uint64_t block = 0; block < nblocks; ++block) { + uint64_t k1 = loadWord(keyData + block * 16); + uint64_t k2 = loadWord(keyData + block * 16 + 8); + k1 *= c1; + k1 = rotl64(k1, 31); + k1 *= c2; + h1 ^= k1; + h1 = rotl64(h1, 27); + h1 += h2; + h1 = h1 * 5 + 0x52dce729; + k2 *= c2; + k2 = rotl64(k2, 33); + k2 *= c1; + h2 ^= k2; + h2 = rotl64(h2, 31); + h2 += h1; + h2 = h2 * 5 + 0x38495ab5; + } + const uint64_t tailLength = keyLength & 15; + if (tailLength != 0) { + read_buffer_ptr tail = keyData + keyLength - 16; + uint64_t k1 = loadWord(tail); + uint64_t k2 = loadWord(tail + 8); + k1 *= c1; + k1 = rotl64(k1, 31); + k1 *= c2; + h1 ^= k1; + k2 *= c2; + k2 = rotl64(k2, 33); + k2 *= c1; + h2 ^= k2; + } + h1 ^= keyLength; + h2 ^= keyLength; + h1 += h2; + h2 += h1; + h1 = fmix64(h1); + h2 = fmix64(h2); + return h1 + h2; + } + + JSONIFIER_INLINE static uint64_t mix64(uint64_t value, uint64_t multiplier) noexcept { +#if JSONIFIER_HAS_INT128 + const __uint128_t product = static_cast<__uint128_t>(value) * static_cast<__uint128_t>(multiplier); + return static_cast(product) ^ static_cast(product >> 64); +#elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_X64 + uint64_t highPart; + const uint64_t lowPart = _umul128(value, multiplier, &highPart); + return lowPart ^ highPart; +#else + const uint64_t lowPart = value * multiplier; + return lowPart ^ ((value >> 32) * (multiplier >> 32)); +#endif + } + + inline void grow() noexcept { + const uint64_t newSlotCount = slotCount == 0 ? initialSlots : uint64_t{ slotCount } * 2; + alloc_buffer oldControls{ internal::move(controls) }; + alloc_buffer oldEntries{ internal::move(entries) }; + const uint64_t oldSlotCount = slotCount; + controls.reserve(newSlotCount); + entries.reserve(newSlotCount); + std::memset(controls.data(), static_cast(emptyControl), newSlotCount); + slotCount = static_cast(newSlotCount); + groupMask = newSlotCount / groupWidth - 1; + for (uint64_t slot = 0; slot < oldSlotCount; ++slot) { + if (oldControls.data()[slot] >= 0) { + place(oldEntries.data()[slot]); + } + } + } + + JSONIFIER_INLINE static uint64_t rotl64(uint64_t value, int32_t shift) noexcept { + return (value << shift) | (value >> (64 - shift)); + } + + JSONIFIER_INLINE static uint64_t fmix64(uint64_t value) noexcept { + value ^= value >> 33; + value *= 0xff51afd7ed558ccdull; + value ^= value >> 33; + value *= 0xc4ceb9fe1a85ec53ull; + value ^= value >> 33; + return value; + } + + alloc_buffer controls{}; + alloc_buffer entries{}; + uint64_t groupMask{}; + uint64_t count{}; + uint32_t slotCount{}; + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/parsing/generic_array.hpp b/deps/jsonifier/include/jsonifier-incl/parsing/generic_array.hpp new file mode 100644 index 000000000000..73d0db30c824 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/parsing/generic_array.hpp @@ -0,0 +1,218 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/parsing/generic_array.hpp + */ +#pragma once + +#include + +namespace jsonifier::internal { + + template struct float_array_shaped_loop { + JSONIFIER_INLINE static generic::error_codes impl(internal::document_state* doc, internal::read_structural_index_ptr position, uint64_t depth, + std::vector& out) noexcept { + const internal::read_buffer_ptr root = doc->root; + const internal::read_structural_index_ptr tapeEnd = doc->tapeEnd; + while (position + 1 < tapeEnd) { + const internal::read_buffer_ptr valueStart = root + *position; + const internal::read_buffer_ptr valueEnd = root + *(position + 1); + if (internal::typeTable[*valueStart] != json_types::number) [[unlikely]] { + doc->cursor = position; + return generic::error_codes::incorrect_type; + } + const internal::read_buffer_ptr parsedEnd = internal::float_parser::template parseFloat(out.emplace_back(), valueStart, valueEnd); + if (!parsedEnd || (parsedEnd != valueEnd && !internal::validPostPrimitiveTable[static_cast(*parsedEnd)])) [[unlikely]] { + doc->cursor = position; + return generic::error_codes::number_error; + } + const uint8_t separator = *valueEnd; + if (separator == internal::uint_c_v<','>) [[likely]] { + position += 2; + continue; + } + if (separator == internal::uint_c_v<']'>) [[likely]] { + doc->cursor = position + 2; + doc->depth = depth - 1; + return generic::error_codes::success; + } + doc->cursor = position + 1; + return generic::error_codes::tape_error; + } + doc->cursor = tapeEnd; + return generic::error_codes::unclosed_container; + } + }; + +} + +namespace jsonifier::generic { + + template class generic_array { + public: + using document_state = internal::document_state; + using value = generic_value; + + class iterator { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = value; + using difference_type = ptrdiff_t; + + JSONIFIER_INLINE iterator() noexcept = default; + + JSONIFIER_INLINE iterator(document_state* docNew, internal::read_structural_index_ptr iterNew, uint64_t depthNew) noexcept + : iter{ iterNew }, doc{ docNew }, depth{ depthNew } { + validateElementStart(); + } + + JSONIFIER_INLINE value operator*() const noexcept { + return error_code == error_codes::success ? value{ doc, iter, depth } : value{ error_code }; + } + + JSONIFIER_INLINE bool operator==(const iterator& other) const noexcept { + return iter == other.iter; + } + + JSONIFIER_INLINE iterator& operator++() noexcept { + if (error_code != error_codes::success) [[unlikely]] { + iter = nullptr; + return *this; + } + error_code = doc->advanceChild(iter, depth, internal::uint_c_v<']'>, iter); + validateElementStart(); + return *this; + } + + protected: + JSONIFIER_INLINE void validateElementStart() noexcept { + if (error_code == error_codes::success && iter && document_state::isCloser(doc->charAt(iter))) [[unlikely]] { + error_code = error_codes::tape_error; + } + } + + internal::read_structural_index_ptr iter{}; + document_state* doc{}; + uint64_t depth{}; + error_codes error_code{ error_codes::success }; + }; + + JSONIFIER_INLINE generic_array() noexcept = default; + + JSONIFIER_INLINE generic_array(document_state* docNew, internal::read_structural_index_ptr openNew, uint64_t depthNew) noexcept + : open{ openNew }, doc{ docNew }, depth{ depthNew } { + } + + JSONIFIER_INLINE generic_array(error_codes errorCodeNew) noexcept : error_code{ errorCodeNew } { + } + + JSONIFIER_INLINE generic_array& operator=(const generic_array& other) noexcept { + open = other.open; + doc = other.doc; + depth = other.depth; + error_code = other.error_code; + return *this; + } + + JSONIFIER_INLINE generic_array(const generic_array& other) noexcept : open{ other.open }, doc{ other.doc }, depth{ other.depth }, error_code{ other.error_code } { + } + + [[nodiscard]] JSONIFIER_INLINE error_codes countElements(uint64_t& out) const noexcept { + if (error_code != error_codes::success) [[unlikely]] { + return error_code; + } + out = 0; + for (auto iter = begin(); iter != end(); ++iter) { + if (const error_codes element_error_code = (*iter).error(); element_error_code != error_codes::success) [[unlikely]] { + return element_error_code; + } + ++out; + } + return error_codes::success; + } + + JSONIFIER_INLINE iterator begin() const noexcept { + if (error_code != error_codes::success) [[unlikely]] { + return iterator{}; + } + document_state* const state = doc; + internal::read_structural_index_ptr position = open + 1; + if (position < state->tapeEnd && state->template charAt>(position)) { + state->cursor = position + 1; + state->depth = depth - 1; + return iterator{}; + } + state->cursor = position; + state->depth = depth; + if (position >= state->tapeEnd) [[unlikely]] { + return iterator{}; + } + return iterator{ state, position, depth }; + } + + template [[nodiscard]] JSONIFIER_INLINE error_codes getFloats(std::vector& out) const noexcept { + if (error_code != error_codes::success) [[unlikely]] { + return error_code; + } + out.clear(); + document_state* const state = doc; + internal::read_structural_index_ptr position = open + 1; + if (position >= state->tapeEnd) [[unlikely]] { + state->cursor = state->tapeEnd; + return error_codes::unclosed_container; + } + state->depth = depth; + if (state->template charAt>(position)) { + state->cursor = position + 1; + state->depth = depth - 1; + return error_codes::success; + } + static constexpr uint64_t shortFractionSliceLimit{ 10 }; + const uint64_t firstSliceLength = position + 1 < state->tapeEnd ? static_cast(*(position + 1) - *position) : shortFractionSliceLimit + 1; + if (firstSliceLength <= shortFractionSliceLimit) { + return internal::float_array_shaped_loop::impl(state, position, depth, out); + } + return internal::float_array_shaped_loop::impl(state, position, depth, out); + } + + [[nodiscard]] JSONIFIER_INLINE error_codes rawJson(string_view& out) const noexcept { + return value{ doc, open, depth - 1 }.rawJson(out); + } + + JSONIFIER_INLINE value atPointer(string_view pointer) const noexcept { + return value{ doc, open, depth - 1 }.atPointer(pointer); + } + + JSONIFIER_INLINE value operator[](uint64_t index) const noexcept { + return at(index); + } + + JSONIFIER_INLINE value at(uint64_t index) const noexcept { + if (error_code != error_codes::success) [[unlikely]] { + return value{ error_code }; + } + for (auto iter = begin(); iter != end(); ++iter) { + if (index-- == 0) { + return *iter; + } + } + return value{ error_codes::index_out_of_bounds }; + } + + JSONIFIER_INLINE error_codes error() const noexcept { + return error_code; + } + + JSONIFIER_INLINE iterator end() const noexcept { + return iterator{}; + } + + protected: + internal::read_structural_index_ptr open{}; + document_state* doc{}; + uint64_t depth{}; + error_codes error_code{ error_codes::success }; + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/parsing/generic_common_post.hpp b/deps/jsonifier/include/jsonifier-incl/parsing/generic_common_post.hpp new file mode 100644 index 000000000000..cf2939aaa653 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/parsing/generic_common_post.hpp @@ -0,0 +1,207 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/parsing/generic_common_post.hpp + */ +#pragma once + +#include + +namespace jsonifier::internal { + + struct generic_storage { + JSONIFIER_INLINE generic_storage() noexcept = default; + + JSONIFIER_INLINE generic_storage& operator=(generic_storage&& other) noexcept { + if (this != &other) [[likely]] { + fieldIndex = internal::move(other.fieldIndex); + arena = internal::move(other.arena); + state = document_state{}; + other.state = document_state{}; + } + return *this; + } + + JSONIFIER_INLINE generic_storage(generic_storage&& other) noexcept : fieldIndex{ internal::move(other.fieldIndex) }, arena{ internal::move(other.arena) } { + other.state = document_state{}; + } + + generic_storage& operator=(const generic_storage&) = delete; + generic_storage(const generic_storage&) = delete; + + JSONIFIER_INLINE document_state& stateFor() noexcept { + return state; + } + + field_index_map fieldIndex{}; + document_state state{}; + string_arena arena{}; + }; + +} + +namespace jsonifier::generic { + + template JSONIFIER_INLINE generic_object generic_value::getObject() const noexcept { + if (error_code != error_codes::success) [[unlikely]] { + return object{ error_code }; + } + if (!doc->template charAt>(iter)) [[unlikely]] { + return object{ error_codes::incorrect_type }; + } + doc->cursor = iter + 1; + doc->depth = depth + 1; + return object{ doc, iter, depth + 1 }; + } + + template JSONIFIER_INLINE generic_array generic_value::getArray() const noexcept { + if (error_code != error_codes::success) [[unlikely]] { + return array{ error_code }; + } + if (!doc->template charAt>(iter)) [[unlikely]] { + return array{ error_codes::incorrect_type }; + } + doc->cursor = iter + 1; + doc->depth = depth + 1; + return array{ doc, iter, depth + 1 }; + } + + template template + JSONIFIER_INLINE generic_value generic_value::operator[](key_type&& key) const noexcept { + return getObject().findFieldUnordered(static_cast(key)); + } + + template JSONIFIER_INLINE generic_value generic_value::operator[](uint64_t index) const noexcept { + return getArray().at(index); + } + + template JSONIFIER_INLINE bool generic_value::unescapePointerToken(string_view& token, string& unescapedToken) noexcept { + if (!internal::char_comparison<'~', char>::memchar(token.data(), token.size())) { + return true; + } + unescapedToken.clear(); + for (uint64_t x = 0; x < token.size(); ++x) { + if (token[x] != '~') { + unescapedToken.emplace_back(token[x]); + } else if (x + 1 < token.size() && (token[x + 1] == '0' || token[x + 1] == '1')) { + unescapedToken.emplace_back(token[x + 1] == '0' ? '~' : '/'); + ++x; + } else { + return false; + } + } + token = unescapedToken; + return true; + } + + template JSONIFIER_INLINE generic_value generic_value::findFieldUnordered(string_view key) const noexcept { + return getObject().findFieldUnordered(key); + } + + template JSONIFIER_INLINE error_codes generic_value::parsePointerIndex(string_view token, uint64_t& index) noexcept { + if (token.empty() || (token.size() > 1 && token[0] == '0')) [[unlikely]] { + return error_codes::invalid_json_pointer; + } + index = 0; + for (const char digit: token) { + if (digit < '0' || digit > '9') [[unlikely]] { + return error_codes::invalid_json_pointer; + } + const uint64_t digitValue = static_cast(digit - '0'); + if (index > (std::numeric_limits::max() - digitValue) / 10) [[unlikely]] { + return error_codes::index_out_of_bounds; + } + index = index * 10 + digitValue; + } + return error_codes::success; + } + + template template JSONIFIER_INLINE error_codes generic_value::get(value_type& out) const { + if constexpr (std::is_same_v) { + return getDouble(out); + } else if constexpr (std::is_same_v) { + return getInt64(out); + } else if constexpr (std::is_same_v) { + return getUint64(out); + } else if constexpr (std::is_same_v) { + return getBool(out); + } else if constexpr (std::is_same_v || internal::buffer_like_types) { + return getString(out); + } else if constexpr (std::is_same_v) { + out = getObject(); + return out.error(); + } else if constexpr (std::is_same_v) { + out = getArray(); + return out.error(); + } else { + static_assert(std::is_same_v, "Unsupported generic::value::get type."); + out = *this; + return error_code; + } + } + + template template JSONIFIER_INLINE error_codes generic_value::getArray(std::vector& out) const { + const array arrayView = getArray(); + if constexpr (internal::float_types) { + return arrayView.getFloats(out); + } else { + if (arrayView.error() != error_codes::success) [[unlikely]] { + return arrayView.error(); + } + out.clear(); + for (auto element: arrayView) { + if constexpr (internal::vector_types) { + if (const error_codes element_error_code = element.getArray(out.emplace_back()); element_error_code != error_codes::success) [[unlikely]] { + return element_error_code; + } + } else { + if (const error_codes element_error_code = element.get(out.emplace_back()); element_error_code != error_codes::success) [[unlikely]] { + return element_error_code; + } + } + } + return error_codes::success; + } + } + + template JSONIFIER_INLINE generic_value generic_value::findField(string_view key) const noexcept { + return getObject().findField(key); + } + + template JSONIFIER_INLINE generic_value generic_value::pointerStep(string_view token, string& unescapedToken) const noexcept { + switch (doc->charAt(iter)) { + case internal::uint_c_v<'{'>: { + return unescapePointerToken(token, unescapedToken) ? findField(token) : value{ error_codes::invalid_json_pointer }; + } + case internal::uint_c_v<'['>: { + uint64_t index{}; + const error_codes index_error_code = parsePointerIndex(token, index); + return index_error_code == error_codes::success ? getArray().at(index) : value{ index_error_code }; + } + default: { + return value{ error_codes::invalid_json_pointer }; + } + } + } + + template JSONIFIER_INLINE generic_value generic_value::atPointer(string_view pointer) const noexcept { + value current{ *this }; + string unescapedToken{}; + const char* position = pointer.data(); + const char* pointerEnd = position + pointer.size(); + while (current.error_code == error_codes::success && position < pointerEnd) { + if (*position != '/') [[unlikely]] { + current = value{ error_codes::invalid_json_pointer }; + } else { + ++position; + const char* slash = internal::char_comparison<'/', char>::memchar(position, static_cast(pointerEnd - position)); + const char* tokenEnd = slash ? slash : pointerEnd; + current = current.pointerStep(string_view{ position, static_cast(tokenEnd - position) }, unescapedToken); + position = tokenEnd; + } + } + return current; + } + +} diff --git a/deps/jsonifier/include/jsonifier-incl/parsing/generic_common_pre.hpp b/deps/jsonifier/include/jsonifier-incl/parsing/generic_common_pre.hpp new file mode 100644 index 000000000000..88c2af7d0d1b --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/parsing/generic_common_pre.hpp @@ -0,0 +1,212 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/parsing/generic_common_pre.hpp + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace jsonifier::internal { + + consteval array genValidPostPrimitiveTable() { + array table{}; + table[static_cast(',')] = true; + table[static_cast('}')] = true; + table[static_cast(']')] = true; + table[static_cast(' ')] = true; + table[static_cast('\t')] = true; + table[static_cast('\n')] = true; + table[static_cast('\r')] = true; + return table; + } + + alignas(64) inline constexpr auto validPostPrimitiveTable{ genValidPostPrimitiveTable() }; + +} + +namespace jsonifier::generic { + + enum class error_codes : uint8_t { + success, + empty, + tape_error, + unclosed_container, + trailing_content, + incorrect_type, + no_such_field, + index_out_of_bounds, + number_error, + string_error, + t_atom_error, + f_atom_error, + n_atom_error, + invalid_json_pointer, + capacity, + count, + }; + + template class generic_object; + template class generic_array; + template class generic_field; + +} + +namespace jsonifier::internal { + + template class alloc_buffer : internal::alloc_wrapper { + public: + using allocator = internal::alloc_wrapper; + + JSONIFIER_INLINE alloc_buffer() noexcept = default; + + JSONIFIER_INLINE explicit alloc_buffer(uint64_t capacityNew) noexcept { + reserve(capacityNew); + } + + JSONIFIER_INLINE alloc_buffer& operator=(alloc_buffer&& other) noexcept { + if (this != &other) [[likely]] { + release(); + ptr = std::exchange(other.ptr, nullptr); + capacity = std::exchange(other.capacity, uint64_t{}); + } + return *this; + } + + JSONIFIER_INLINE alloc_buffer(alloc_buffer&& other) noexcept : capacity{ std::exchange(other.capacity, uint64_t{}) }, ptr{ std::exchange(other.ptr, nullptr) } { + } + + alloc_buffer& operator=(const alloc_buffer&) = delete; + alloc_buffer(const alloc_buffer&) = delete; + + JSONIFIER_INLINE value_type* data() const noexcept { + return ptr; + } + + inline void reserve(uint64_t capacityNew) noexcept { + if (capacityNew > capacity) { + release(); + ptr = allocator::allocate(capacityNew); + capacity = capacityNew; + } + } + + JSONIFIER_INLINE uint64_t size() const noexcept { + return capacity; + } + + JSONIFIER_INLINE ~alloc_buffer() noexcept { + release(); + } + + protected: + JSONIFIER_INLINE void release() noexcept { + if (ptr) { + allocator::deallocate(ptr, capacity); + ptr = nullptr; + capacity = 0; + } + } + + uint64_t capacity{}; + value_type* ptr{}; + }; + + struct string_arena { + inline string_arena() noexcept = default; + + JSONIFIER_INLINE string_arena& operator=(string_arena&& other) noexcept { + if (this != &other) [[likely]] { + buffer = internal::move(other.buffer); + scratchPtr = std::exchange(other.scratchPtr, nullptr); + slots = std::exchange(other.slots, nullptr); + } + return *this; + } + + JSONIFIER_INLINE string_arena(string_arena&& other) noexcept + : buffer{ internal::move(other.buffer) }, scratchPtr{ std::exchange(other.scratchPtr, nullptr) }, slots{ std::exchange(other.slots, nullptr) } { + } + + string_arena& operator=(const string_arena&) = delete; + string_arena(const string_arena&) = delete; + + JSONIFIER_INLINE void reset(uint64_t documentLength) noexcept { + buffer.reserve(documentLength * 2 + 128); + slots = buffer.data(); + scratchPtr = buffer.data() + documentLength + 64; + } + + JSONIFIER_INLINE char* slot(uint64_t offset) const noexcept { + return slots + offset; + } + + JSONIFIER_INLINE char* scratch() const noexcept { + return scratchPtr; + } + + protected: + alloc_buffer buffer{}; + char* scratchPtr{}; + char* slots{}; + }; + + struct utf8_span_step { + template + JSONIFIER_INLINE static unrestrict_t impl(unrestrict_t& current, read_buffer_ptr end, internal::utf8_validation_state& state) noexcept { + using simd_list_local = type_list_element_t; + static constexpr auto avx_type = simd_list_local::type::avx_type; + using simd_type_local = typename simd_type_wrapper::type; + static constexpr uint64_t bytesProcessed = simd_list_local::bytesProcessed; + internal::utf8_register_validator> validator{ state }; + while (static_cast(end - current) >= bytesProcessed) { + validator.checkRegister(simd::gatherValuesU(current)); + current += bytesProcessed; + } + if constexpr (index == listSize - 1) { + validator.checkPartial(current, static_cast(end - current)); + current = end; + return validator.errors() ? nullptr : current; + } else { + validator.flush(); + return current; + } + } + }; + + struct key_matcher { + template JSONIFIER_INLINE static bool impl(state_type&& state, key_type&& key, uint32_t keyIndex) noexcept { + return state->template keyMatches(state->tape + keyIndex, key); + } + }; + + struct document_state; + class field_index_map; + + struct plain_string_copy_context { + plain_string_copy_context() noexcept = delete; + + JSONIFIER_INLINE plain_string_copy_context(read_buffer_ptr sourceNew) noexcept : source{ sourceNew } { + } + + plain_string_copy_context& operator=(plain_string_copy_context&&) noexcept = delete; + plain_string_copy_context(plain_string_copy_context&&) noexcept = delete; + plain_string_copy_context& operator=(const plain_string_copy_context&) noexcept = delete; + plain_string_copy_context(const plain_string_copy_context&) noexcept = delete; + + JSONIFIER_INLINE uint64_t operator()(char* __restrict ptrNew, uint64_t length) const noexcept { + jsonifierMemcpy(ptrNew, source, length); + return length; + } + + read_buffer_ptr source{}; + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/parsing/generic_document.hpp b/deps/jsonifier/include/jsonifier-incl/parsing/generic_document.hpp new file mode 100644 index 000000000000..c3292f493eda --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/parsing/generic_document.hpp @@ -0,0 +1,23 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/parsing/generic_document.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::generic { + + template class generic_document : public generic_value { + public: + using generic_value::generic_value; + + JSONIFIER_INLINE bool atEnd() const noexcept { + return this->error_code == error_codes::success && this->doc->depth == 0 && this->doc->cursor == this->doc->tapeEnd; + } + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/parsing/generic_field.hpp b/deps/jsonifier/include/jsonifier-incl/parsing/generic_field.hpp new file mode 100644 index 000000000000..b8abbd393e3a --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/parsing/generic_field.hpp @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/parsing/generic_field.hpp + */ +#pragma once + +#include + +namespace jsonifier::generic { + + template class generic_field { + public: + using document_state = internal::document_state; + + JSONIFIER_INLINE generic_field() noexcept = default; + + JSONIFIER_INLINE generic_field(document_state* docNew, internal::read_structural_index_ptr iterNew, uint64_t depthNew, + error_codes errorCodeNew = error_codes::success) noexcept + : iter{ iterNew }, doc{ docNew }, depth{ depthNew }, error_code{ errorCodeNew } { + } + + JSONIFIER_INLINE generic_field& operator=(const generic_field& other) noexcept { + iter = other.iter; + doc = other.doc; + depth = other.depth; + error_code = other.error_code; + return *this; + } + + JSONIFIER_INLINE generic_field(const generic_field& other) noexcept : iter{ other.iter }, doc{ other.doc }, depth{ other.depth }, error_code{ other.error_code } { + } + + JSONIFIER_INLINE generic_value value() const noexcept { + return error_code == error_codes::success ? generic_value{ doc, iter + 2, depth } : generic_value{ error_code }; + } + + JSONIFIER_INLINE string_view key() const noexcept { + internal::read_buffer_ptr keyStart = doc->root + *iter + 1; + internal::read_buffer_ptr keyEnd = doc->root + *(iter + 1) - 1; + if constexpr (!minified) { + while (keyEnd > keyStart && *keyEnd != internal::uint_c_v<'"'>) { + --keyEnd; + } + } + return string_view{ std::bit_cast(keyStart), static_cast(keyEnd - keyStart) }; + } + + [[nodiscard]] JSONIFIER_INLINE error_codes unescapedKey(std::string_view& out) const noexcept { + return generic_value{ doc, iter, depth }.unescape(iter, out); + } + + JSONIFIER_INLINE bool keyEquals(string_view keyNew) const noexcept { + return doc->template keyMatches(iter, keyNew); + } + + JSONIFIER_INLINE string_view rawKey() const noexcept { + return doc->template rawKey(iter); + } + + JSONIFIER_INLINE error_codes error() const noexcept { + return error_code; + } + + protected: + internal::read_structural_index_ptr iter{}; + document_state* doc{}; + uint64_t depth{}; + error_codes error_code{ error_codes::success }; + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/parsing/generic_iterator.hpp b/deps/jsonifier/include/jsonifier-incl/parsing/generic_iterator.hpp new file mode 100644 index 000000000000..1a00f66d63c3 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/parsing/generic_iterator.hpp @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/parsing/generic_iterator.hpp + */ +#pragma once + +#include + +namespace jsonifier::internal { + + template struct generic_iterator { + using derived_type = derived_type_new; + + static constexpr uint64_t smallDocumentBytes{ simdBytesPerStep }; + + template generic::generic_document iterate(const buffer_type& in) { + using generic::error_codes; + using document = generic::generic_document; + read_buffer_ptr rootIter = std::bit_cast(in.data()); + const uint64_t length = in.size(); + if (!rootIter || length == 0) [[unlikely]] { + return document{ error_codes::empty }; + } + const uint64_t tapeCount = indexGenericWindow(rootIter, length); + if (tapeCount == 0) [[unlikely]] { + return document{ error_codes::empty }; + } + generic_storage& data = derivedRef.genericStorage(); + document_state& state = data.stateFor(); + state.root = rootIter; + state.end = rootIter + length; + state.cursor = state.tape; + state.depth = 0; + if (tapeCount > 1 && !document_state::isOpener(rootIter[state.tape[0]])) [[unlikely]] { + return document{ error_codes::trailing_content }; + } + data.arena.reset(length); + data.fieldIndex.reset(); + state.arena = &data.arena; + state.fieldIndex = &data.fieldIndex; + return document{ &state, state.tape, 0 }; + } + + template inline uint64_t indexGenericWindow(read_buffer_ptr rootIter, uint64_t length) noexcept { + document_state& state = derivedRef.genericStorage().stateFor(); + if (length < smallDocumentBytes) { + derivedRef.podSection.template reset(rootIter, length); + state.tape = derivedRef.podSection.begin(); + state.tapeEnd = derivedRef.podSection.end(); + return derivedRef.podSection.getTapeCount(); + } + derivedRef.section.template reset(rootIter, length); + state.tape = derivedRef.section.begin(); + state.tapeEnd = derivedRef.section.end(); + return derivedRef.section.getTapeCount(); + } + + protected: + inline generic_iterator() noexcept = default; + generic_iterator& operator=(generic_iterator&& other) = delete; + generic_iterator(generic_iterator&& other) = delete; + generic_iterator& operator=(const generic_iterator& other) = delete; + generic_iterator(const generic_iterator& other) = delete; + inline ~generic_iterator() noexcept = default; + + derived_type& derivedRef{ *static_cast(this) }; + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/parsing/generic_object.hpp b/deps/jsonifier/include/jsonifier-incl/parsing/generic_object.hpp new file mode 100644 index 000000000000..88068bc53d5a --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/parsing/generic_object.hpp @@ -0,0 +1,464 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/parsing/generic_object.hpp + */ +#pragma once + +#include + +namespace jsonifier::generic { + + template class generic_object { + public: + using document_state = internal::document_state; + using field = generic_field; + using value = generic_value; + + class iterator { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = field; + using difference_type = ptrdiff_t; + + JSONIFIER_INLINE iterator() noexcept = default; + + JSONIFIER_INLINE iterator(document_state* docNew, internal::read_structural_index_ptr iterNew, uint64_t depthNew) noexcept + : iter{ iterNew }, doc{ docNew }, depth{ depthNew } { + validateFieldStart(); + } + + JSONIFIER_INLINE iterator& operator++() noexcept { + if (error_code != error_codes::success) [[unlikely]] { + iter = nullptr; + return *this; + } + error_code = doc->advanceChild(iter + 2, depth, internal::uint_c_v<'}'>, iter); + validateFieldStart(); + return *this; + } + + JSONIFIER_INLINE bool operator==(const iterator& other) const noexcept { + return iter == other.iter; + } + + JSONIFIER_INLINE field operator*() const noexcept { + return field{ doc, iter, depth, error_code }; + } + + protected: + JSONIFIER_INLINE void validateFieldStart() noexcept { + if (error_code == error_codes::success && iter && + (!doc->template charAt>(iter) || iter + 1 >= doc->tapeEnd || !doc->template charAt>(iter + 1))) [[unlikely]] { + error_code = error_codes::tape_error; + } + } + + internal::read_structural_index_ptr iter{}; + document_state* doc{}; + uint64_t depth{}; + error_codes error_code{ error_codes::success }; + }; + +#if JSONIFIER_PLATFORM_MAC && JSONIFIER_COMPILER_CLANG + static constexpr uint32_t indexThreshold{ 16 }; +#else + static constexpr uint32_t indexThreshold{ 8 }; +#endif + static constexpr uint32_t wrapThreshold{ 2 }; + static constexpr uint32_t trailCapacity{ 16 }; + + JSONIFIER_INLINE generic_object() noexcept = default; + + JSONIFIER_INLINE generic_object(document_state* docNew, internal::read_structural_index_ptr openNew, uint64_t depthNew) noexcept + : open{ openNew }, doc{ docNew }, depth{ depthNew } { + } + + JSONIFIER_INLINE generic_object(error_codes errorCodeNew) noexcept : error_code{ errorCodeNew } { + } + + [[nodiscard]] JSONIFIER_INLINE error_codes countFields(uint64_t& out) const noexcept { + if (error_code != error_codes::success) [[unlikely]] { + return error_code; + } + out = 0; + for (auto iter = begin(); iter != end(); ++iter) { + if (const error_codes field_error_code = (*iter).error(); field_error_code != error_codes::success) [[unlikely]] { + return field_error_code; + } + ++out; + } + return error_codes::success; + } + + JSONIFIER_INLINE iterator begin() const noexcept { + if (error_code != error_codes::success) [[unlikely]] { + return iterator{}; + } + document_state* const state = doc; + internal::read_structural_index_ptr position = open + 1; + if (position < state->tapeEnd && state->template charAt>(position)) { + state->cursor = position + 1; + state->depth = depth - 1; + return iterator{}; + } + state->cursor = position; + state->depth = depth; + if (position >= state->tapeEnd) [[unlikely]] { + return iterator{}; + } + return iterator{ state, position, depth }; + } + + [[nodiscard]] JSONIFIER_INLINE error_codes rawJson(string_view& out) const noexcept { + return value{ doc, open, depth - 1 }.rawJson(out); + } + + JSONIFIER_INLINE value atPointer(string_view pointer) const noexcept { + return value{ doc, open, depth - 1 }.atPointer(pointer); + } + + JSONIFIER_INLINE value findFieldUnordered(string_view key) noexcept { + value found{ error_code }; + if (error_code == error_codes::success) [[likely]] { + settle(); + found = expectedField(key); + if (found.error_code == error_codes::no_such_field) [[unlikely]] { + found = indexedField(key); + } + } + return found; + } + + JSONIFIER_INLINE value operator[](string_view key) noexcept { + return findFieldUnordered(key); + } + + JSONIFIER_INLINE value findField(string_view key) noexcept { + value found{ error_code }; + if (error_code == error_codes::success) [[likely]] { + settle(); + found = scanFields(key, doc->tapeEnd); + if (found.error_code == error_codes::no_such_field) [[unlikely]] { + found = escapeAwareRetry(key, doc->cursor); + } + } + return found; + } + + JSONIFIER_INLINE error_codes error() const noexcept { + return error_code; + } + + JSONIFIER_INLINE iterator end() const noexcept { + return iterator{}; + } + + protected: + JSONIFIER_INLINE void recordField(internal::read_structural_index_ptr keyIter) const noexcept { + document_state* const state = doc; + internal::read_buffer_ptr keyStart = state->root + *keyIter + 1; + internal::read_buffer_ptr keyEnd = state->root + *(keyIter + 1) - 1; + if constexpr (!minified) { + while (keyEnd > keyStart && *keyEnd != internal::uint_c_v<'"'>) { + --keyEnd; + } + } + const uint32_t object = objectIndex(); + state->fieldIndex->insert(internal::field_index_map::hashKey(keyStart, static_cast(keyEnd - keyStart), object), object, + static_cast(keyIter - state->tape)); + } + + JSONIFIER_INLINE value expectedField(string_view key) const noexcept { + document_state* const state = doc; + internal::read_structural_index_ptr candidate = state->cursor; + if (candidate < state->tapeEnd && state->template charAt>(candidate)) { + ++candidate; + } + if (candidate + 1 < state->tapeEnd && state->template charAt>(candidate) && state->template charAt>(candidate + 1) && + state->template keyMatches(candidate, key)) [[likely]] { + state->cursor = candidate + 2; + return value{ state, candidate + 2, depth }; + } + return value{ error_codes::no_such_field }; + } + + JSONIFIER_INLINE value extendTrail(string_view key, internal::read_structural_index_ptr resumePoint, bool resumed) noexcept { + document_state* const state = doc; + internal::read_structural_index_ptr tapeEnd = state->tapeEnd; + internal::read_structural_index_ptr cursor = trailCount == 0 ? open + 1 : state->skipValue(state->tape + trailEntry(trailCount - 1) + 2); + while (cursor < tapeEnd) { + const uint8_t current = state->charAt(cursor); + if (current == internal::uint_c_v<'}'>) { + trailComplete = true; + return value{ error_codes::no_such_field }; + } + if (current == internal::uint_c_v<','>) { + ++cursor; + continue; + } + if (current == internal::uint_c_v<'"'> && cursor + 1 < tapeEnd && state->template charAt>(cursor + 1)) [[likely]] { + if (trailCount == trailCapacity) { + trailOverflow = true; + return value{ error_codes::no_such_field }; + } + const uint32_t keyOffset = static_cast(cursor - state->tape); + internal::pow2MemcpyWrapper(trailBytes + trailCount * sizeof(uint32_t), &keyOffset); + ++trailCount; + if (state->template keyMatches(cursor, key)) { + return trailHit(cursor, resumePoint, resumed); + } + cursor += 2; + } + cursor = state->skipValue(cursor); + } + trailOverflow = true; + return value{ error_codes::no_such_field }; + } + + JSONIFIER_INLINE void indexAllFields() const noexcept { + document_state* const state = doc; + internal::read_structural_index_ptr cursor = open + 1; + internal::read_structural_index_ptr tapeEnd = state->tapeEnd; + while (cursor < tapeEnd) { + const uint8_t current = state->charAt(cursor); + if (current == internal::uint_c_v<'}'>) { + return; + } + if (current == internal::uint_c_v<','>) { + ++cursor; + continue; + } + if (current == internal::uint_c_v<'"'> && cursor + 1 < tapeEnd && state->template charAt>(cursor + 1)) [[likely]] { + recordField(cursor); + cursor += 2; + } + cursor = state->skipValue(cursor); + } + } + + template JSONIFIER_INLINE value scanFields(string_view key, internal::read_structural_index_ptr stop) noexcept { + document_state* const state = doc; + internal::read_structural_index_ptr cursor = state->cursor; + internal::read_structural_index_ptr tapeEnd = state->tapeEnd; + uint32_t skipped{ skippedKeys }; + bool expectSeparator{}; + while (cursor < stop) { + const uint8_t current = state->charAt(cursor); + if (current == internal::uint_c_v<','>) { + ++cursor; + expectSeparator = false; + continue; + } + if (current == internal::uint_c_v<'}'>) { + state->cursor = cursor; + skippedKeys = skipped; + return value{ error_codes::no_such_field }; + } + if (expectSeparator) [[unlikely]] { + state->cursor = cursor; + skippedKeys = skipped; + return value{ error_codes::tape_error }; + } + if (current == internal::uint_c_v<'"'> && cursor + 1 < tapeEnd && state->template charAt>(cursor + 1)) [[likely]] { + internal::read_structural_index_ptr valueIter = cursor + 2; + bool matched{}; + if constexpr (escapeAware) { + const bool hasEscape = state->keyHasEscape(cursor); + escapedKeySeen |= hasEscape; + matched = hasEscape && state->unescapedKeyMatches(cursor, key); + } else { + matched = state->template keyMatches(cursor, key); + } + if (matched) { + state->cursor = valueIter; + skippedKeys = skipped; + return value{ state, valueIter, depth }; + } + ++skipped; + cursor = valueIter; + } + cursor = state->skipValue(cursor); + expectSeparator = true; + } + state->cursor = cursor; + skippedKeys = skipped; + return value{ cursor >= tapeEnd ? error_codes::unclosed_container : error_codes::no_such_field }; + } + + JSONIFIER_INLINE value trailHit(internal::read_structural_index_ptr keyIter, internal::read_structural_index_ptr resumePoint, bool resumed) const noexcept { + if (resumed) { + if (keyIter < resumePoint) { + doc->noteBackwardHit(); + } else { + doc->noteForwardHit(); + } + } + doc->cursor = keyIter + 2; + return value{ doc, keyIter + 2, depth }; + } + + JSONIFIER_INLINE value trailLookup(string_view key, internal::read_structural_index_ptr resumePoint, bool resumed) noexcept { + document_state* const state = doc; + for (uint32_t x = 0; x < trailCount; ++x) { + internal::read_structural_index_ptr keyIter = state->tape + trailEntry(x); + if (state->template keyMatches(keyIter, key)) { + return trailHit(keyIter, resumePoint, resumed); + } + } + if (!trailComplete) { + value found{ extendTrail(key, resumePoint, resumed) }; + if (found.error_code == error_codes::success || trailOverflow) { + return found; + } + } + if (!escapeFree) { + bool sawEscape{}; + for (uint32_t x = 0; x < trailCount; ++x) { + internal::read_structural_index_ptr keyIter = state->tape + trailEntry(x); + if (state->keyHasEscape(keyIter)) { + sawEscape = true; + if (state->unescapedKeyMatches(keyIter, key)) { + return trailHit(keyIter, resumePoint, resumed); + } + } + } + escapeFree = !sawEscape; + } + state->cursor = resumePoint; + return value{ error_codes::no_such_field }; + } + + JSONIFIER_INLINE value indexedField(string_view key) noexcept { + document_state* const state = doc; + internal::read_structural_index_ptr resumePoint = state->cursor; + value found{ error_codes::no_such_field }; + if (!indexed) { + if (!indexEarned) { + const bool resumed = resumePoint != open + 1; + if (!trailOverflow) { + if (trailCount == 0 && !trailComplete && !(resumed && state->reverseMode())) { + found = scanFields(key, state->tapeEnd); + if (found.error_code != error_codes::no_such_field) { + if (found.error_code == error_codes::success && resumed) { + state->noteForwardHit(); + } + return found; + } + } + found = trailLookup(key, resumePoint, resumed); + if (!trailOverflow) { + return found; + } + state->cursor = resumePoint; + } + if (resumed && state->reverseMode()) { + found = scanBeforeResume(key, resumePoint); + if (found.error_code == error_codes::no_such_field) { + state->cursor = resumePoint; + found = scanFields(key, state->tapeEnd); + if (found.error_code == error_codes::success) { + state->noteForwardHit(); + } + } + } else { + found = scanFields(key, state->tapeEnd); + if (found.error_code == error_codes::success) { + if (resumed) { + state->noteForwardHit(); + } + } else if (found.error_code == error_codes::no_such_field && resumed) { + found = scanBeforeResume(key, resumePoint); + } + } + if (found.error_code == error_codes::no_such_field) { + found = escapeAwareRetry(key, resumePoint); + } + return found; + } + indexAllFields(); + indexed = true; + } + + const uint32_t object = objectIndex(); + const uint64_t hash = internal::field_index_map::hashKey(std::bit_cast(key.data()), key.size(), object); + const uint32_t hit = state->fieldIndex->template find(hash, object, state, key); + if (hit != internal::field_index_map::npos) { + state->cursor = state->tape + hit + 2; + found = value{ state, state->cursor, depth }; + } + if (found.error_code == error_codes::no_such_field) { + found = escapeAwareRetry(key, resumePoint); + } + return found; + } + + JSONIFIER_INLINE value escapeAwareRetry(string_view key, internal::read_structural_index_ptr restorePoint) noexcept { + if (escapeFree) { + doc->cursor = restorePoint; + return value{ error_codes::no_such_field }; + } + doc->cursor = open + 1; + escapedKeySeen = false; + value found{ scanFields(key, doc->tapeEnd) }; + if (found.error_code == error_codes::no_such_field) { + escapeFree = !escapedKeySeen; + doc->cursor = restorePoint; + } + return found; + } + + JSONIFIER_INLINE value scanBeforeResume(string_view key, internal::read_structural_index_ptr resumePoint) noexcept { + doc->cursor = open + 1; + value found{ scanFields(key, resumePoint) }; + indexEarned = ++wrapCount >= wrapThreshold && skippedKeys >= indexThreshold; + if (found.error_code == error_codes::success) { + doc->noteBackwardHit(); + } + return found; + } + + JSONIFIER_INLINE uint32_t trailEntry(uint32_t index) const noexcept { + uint32_t keyOffset; + internal::pow2MemcpyWrapper(&keyOffset, trailBytes + index * sizeof(uint32_t)); + return keyOffset; + } + + JSONIFIER_INLINE uint32_t objectIndex() const noexcept { + return static_cast(open - doc->tape); + } + + JSONIFIER_INLINE void settle() const noexcept { + if (doc->depth != depth) [[unlikely]] { + if (doc->depth > depth) { + doc->finishToDepth(depth); + } + if (doc->depth != depth) { + rewind(); + } + } + } + + JSONIFIER_INLINE void rewind() const noexcept { + doc->cursor = open + 1; + doc->depth = depth; + } + + internal::read_structural_index_ptr open{}; + document_state* doc{}; + uint64_t depth{}; + uint32_t skippedKeys{}; + uint32_t trailCount{}; + uint32_t wrapCount{}; + uint8_t trailBytes[trailCapacity * sizeof(uint32_t)]; + error_codes error_code{ error_codes::success }; + bool escapedKeySeen{}; + bool trailComplete{}; + bool trailOverflow{}; + bool indexEarned{}; + bool escapeFree{}; + bool indexed{}; + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/parsing/generic_tables.hpp b/deps/jsonifier/include/jsonifier-incl/parsing/generic_tables.hpp new file mode 100644 index 000000000000..29c27d84e807 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/parsing/generic_tables.hpp @@ -0,0 +1,102 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/parsing/generic_tables.hpp + */ +#pragma once + +#include + +namespace jsonifier::internal { + + // Sampled from Dr. Lemire's library, simdjson: https://github.com/simdjson/simdjson + consteval internal::array genDepthDeltaTable() noexcept { + internal::array returnValues{}; + for (auto& entry: returnValues) { + entry = 0; + } + returnValues[uint_c_v<'{'>] = 1; + returnValues[uint_c_v<'['>] = 1; + returnValues[uint_c_v<'}'>] = -1; + returnValues[uint_c_v<']'>] = -1; + return returnValues; + } + + alignas(64) inline constexpr auto depthDeltaTableRaw{ genDepthDeltaTable() }; + + inline constexpr const int8_t* __restrict depthDeltaTable{ depthDeltaTableRaw.data() }; + + using array_type_table = internal::array; + + consteval array_type_table genTypeTable() noexcept { + array_type_table returnValues{}; + for (auto& entry: returnValues) { + entry = json_types::unset; + } + returnValues[uint_c_v<'{'>] = json_types::object; + returnValues[uint_c_v<'['>] = json_types::array; + returnValues[uint_c_v<'"'>] = json_types::string; + returnValues[uint_c_v<'t'>] = json_types::boolean; + returnValues[uint_c_v<'f'>] = json_types::boolean; + returnValues[uint_c_v<'n'>] = json_types::null; + returnValues[uint_c_v<'-'>] = json_types::number; + for (uint64_t x = uint_c_v<'0'>; x <= uint_c_v<'9'>; ++x) { + returnValues[x] = json_types::number; + } + return returnValues; + } + + alignas(64) inline constexpr auto typeTableRaw{ genTypeTable() }; + + inline constexpr const json_types* __restrict typeTable{ typeTableRaw.data() }; + + consteval internal::array genSeperatorByteTable() noexcept { + internal::array returnValues{}; + for (auto& entry: returnValues) { + entry = false; + } + returnValues[uint_c_v<' '>] = true; + returnValues[uint_c_v<'\n'>] = true; + returnValues[uint_c_v<'\r'>] = true; + returnValues[uint_c_v<'\t'>] = true; + returnValues[uint_c_v<','>] = true; + return returnValues; + } + + alignas(64) inline constexpr auto seperatorByteTableRaw{ genSeperatorByteTable() }; + + inline constexpr const bool* __restrict seperatorByteTable{ seperatorByteTableRaw.data() }; + + consteval internal::array genUtf8StringStopTable() noexcept { + internal::array returnValues{}; + for (uint64_t x = 0; x < 0x20; ++x) { + returnValues[x] = true; + } + returnValues[uint_c_v<'"'>] = true; + returnValues[uint_c_v<'\\'>] = true; + return returnValues; + } + + alignas(64) inline constexpr auto utf8StringStopTableRaw{ genUtf8StringStopTable() }; + + inline constexpr const bool* __restrict utf8StringStopTable{ utf8StringStopTableRaw.data() }; + + consteval internal::array genAsciiStringStopTable() noexcept { + internal::array returnValues{}; + for (uint64_t x = 0; x < 0x20; ++x) { + returnValues[x] = true; + } + for (uint64_t x = 0x80; x < 0x100; ++x) { + returnValues[x] = true; + } + returnValues[uint_c_v<'"'>] = true; + returnValues[uint_c_v<'\\'>] = true; + return returnValues; + } + + alignas(64) inline constexpr auto asciiStringStopTableRaw{ genAsciiStringStopTable() }; + + inline constexpr const bool* __restrict asciiStringStopTable{ asciiStringStopTableRaw.data() }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/parsing/generic_value.hpp b/deps/jsonifier/include/jsonifier-incl/parsing/generic_value.hpp new file mode 100644 index 000000000000..bf02a7768435 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/parsing/generic_value.hpp @@ -0,0 +1,394 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/parsing/generic_value.hpp + */ +#pragma once + +#include + +namespace jsonifier::generic { + + template class generic_value { + public: + using document_state = internal::document_state; + using object = generic_object; + using array = generic_array; + using value = generic_value; + + JSONIFIER_INLINE generic_value() noexcept = default; + + JSONIFIER_INLINE generic_value(document_state* docNew, internal::read_structural_index_ptr iterNew, uint64_t depthNew) noexcept + : iter{ iterNew }, doc{ docNew }, depth{ depthNew } { + } + + JSONIFIER_INLINE generic_value(error_codes errorCodeNew) noexcept : error_code{ errorCodeNew } { + } + + JSONIFIER_INLINE generic_value& operator=(const generic_value& other) noexcept { + iter = other.iter; + doc = other.doc; + depth = other.depth; + error_code = other.error_code; + return *this; + } + + JSONIFIER_INLINE generic_value(const generic_value& other) noexcept : iter{ other.iter }, doc{ other.doc }, depth{ other.depth }, error_code{ other.error_code } { + } + + template JSONIFIER_INLINE value operator[](key_type&& key) const noexcept; + + [[nodiscard]] JSONIFIER_INLINE error_codes rawJson(string_view& out) const noexcept { + if (error_code != error_codes::success) [[unlikely]] { + return error_code; + } + internal::read_buffer_ptr valueStart = doc->root + *iter; + if (document_state::isOpener(*valueStart)) { + internal::read_structural_index_ptr afterClose = doc->skipValue(iter); + if (afterClose == doc->tapeEnd && !document_state::isCloser(doc->charAt(afterClose - 1))) [[unlikely]] { + return error_codes::unclosed_container; + } + if (doc->cursor == iter) [[likely]] { + doc->cursor = afterClose; + } + out = string_view{ std::bit_cast(valueStart), static_cast(doc->root + *(afterClose - 1) + 1 - valueStart) }; + return error_codes::success; + } + internal::read_buffer_ptr valueEnd = doc->root + *(iter + 1); + while (valueEnd > valueStart && isWhitespace(valueEnd[-1])) { + --valueEnd; + } + out = string_view{ std::bit_cast(valueStart), static_cast(valueEnd - valueStart) }; + return error_codes::success; + } + + template [[nodiscard]] JSONIFIER_INLINE error_codes getString(string_type& out) const { + if (error_code != error_codes::success) [[unlikely]] { + return error_code; + } + if (!doc->template charAt>(iter)) [[unlikely]] { + return error_codes::incorrect_type; + } + consume(); + internal::read_buffer_ptr stringStart = doc->root + *iter + 1; + if (const uint64_t directLength = zeroCopyLength(stringStart); directLength != noZeroCopy) [[likely]] { + if constexpr (internal::resize_and_overwriteable_types) { + out.resize_and_overwrite(directLength, internal::plain_string_copy_context{ stringStart }); + } else { + out.resize(directLength); + internal::plain_string_copy_context{ stringStart }(out.data(), directLength); + } + return error_codes::success; + } + const uint64_t upperBound = static_cast(doc->root + *(iter + 1) - stringStart) + 64; + uint64_t outLength{}; + if constexpr (internal::resize_and_overwriteable_types) { + out.resize_and_overwrite(upperBound, internal::string_parse_context{ stringStart, doc->end, outLength }); + } else { + if (out.size() < upperBound) [[unlikely]] { + out.resize(upperBound); + } + out.resize(internal::string_parse_context{ stringStart, doc->end, outLength }(out.data(), upperBound)); + } + return outLength == std::numeric_limits::max() ? error_codes::string_error : error_codes::success; + } + + [[nodiscard]] JSONIFIER_INLINE bool isNull() const noexcept { + if (error_code == error_codes::success && doc->template charAt>(iter) && nullMatches()) { + consume(); + return true; + } + return false; + } + + [[nodiscard]] JSONIFIER_INLINE error_codes getUint64(uint64_t& out) const noexcept { + if (!isNumberStart()) [[unlikely]] { + return typeError(); + } + internal::read_buffer_ptr valueStart = doc->root + *iter; + internal::read_buffer_ptr valueEnd = doc->root + *(iter + 1); + internal::read_buffer_ptr parsedEnd = internal::integer_parser::parseInt(out, valueStart, valueEnd); + consume(); + return finishNumber(parsedEnd, valueEnd); + } + + [[nodiscard]] JSONIFIER_INLINE error_codes getInt64(int64_t& out) const noexcept { + if (!isNumberStart()) [[unlikely]] { + return typeError(); + } + internal::read_buffer_ptr valueStart = doc->root + *iter; + internal::read_buffer_ptr valueEnd = doc->root + *(iter + 1); + internal::read_buffer_ptr parsedEnd = internal::integer_parser::parseInt(out, valueStart, valueEnd); + consume(); + return finishNumber(parsedEnd, valueEnd); + } + + [[nodiscard]] JSONIFIER_INLINE error_codes getDouble(double& out) const noexcept { + if (!isNumberStart()) [[unlikely]] { + return typeError(); + } + internal::read_buffer_ptr valueStart = doc->root + *iter; + internal::read_buffer_ptr valueEnd = doc->root + *(iter + 1); + internal::read_buffer_ptr parsedEnd = internal::float_parser::parseFloat(out, valueStart, valueEnd); + consume(); + return finishNumber(parsedEnd, valueEnd); + } + + template [[nodiscard]] JSONIFIER_INLINE error_codes get(value_type& out) const; + + template [[nodiscard]] JSONIFIER_INLINE error_codes getArray(std::vector& out) const; + + [[nodiscard]] JSONIFIER_INLINE error_codes getString(std::string_view& out) const noexcept { + if (error_code != error_codes::success) [[unlikely]] { + return error_code; + } + if (!doc->template charAt>(iter)) [[unlikely]] { + return error_codes::incorrect_type; + } + consume(); + return unescape(iter, out); + } + + [[nodiscard]] JSONIFIER_INLINE error_codes getBool(bool& out) const noexcept { + if (error_code != error_codes::success) [[unlikely]] { + return error_code; + } + switch (doc->charAt(iter)) { + case internal::uint_c_v<'t'>: { + consume(); + out = true; + return trueMatches() ? error_codes::success : error_codes::t_atom_error; + } + case internal::uint_c_v<'f'>: { + consume(); + out = false; + return falseMatches() ? error_codes::success : error_codes::f_atom_error; + } + default: { + return error_codes::incorrect_type; + } + } + } + + JSONIFIER_INLINE value findFieldUnordered(string_view key) const noexcept; + + JSONIFIER_INLINE value atPointer(string_view pointer) const noexcept; + + JSONIFIER_INLINE value findField(string_view key) const noexcept; + + JSONIFIER_INLINE value operator[](uint64_t index) const noexcept; + + JSONIFIER_INLINE error_codes error() const noexcept { + return error_code; + } + + JSONIFIER_INLINE object getObject() const noexcept; + + JSONIFIER_INLINE json_types type() const noexcept { + if (error_code != error_codes::success) [[unlikely]] { + return json_types::unset; + } + return internal::typeTable[doc->charAt(iter)]; + } + + JSONIFIER_INLINE array getArray() const noexcept; + + protected: + static constexpr uint64_t noZeroCopy{ std::numeric_limits::max() }; +#if JSONIFIER_NEON + static constexpr uint64_t stopMaskLaneShift{ 2 }; +#else + static constexpr uint64_t stopMaskLaneShift{ 0 }; +#endif + + struct string_stop_masks { + uint64_t nonAscii{}; + uint64_t stops{}; + }; + + friend class generic_object; + friend class generic_array; + friend class generic_field; + + JSONIFIER_INLINE static bool validUtf8(internal::read_buffer_ptr source, uint64_t length) noexcept { + internal::utf8_validation_state state{}; + internal::unrestrict_t current = source; + return internal::string_parse_executor>::impl(current, + source + length, state); + } + + template JSONIFIER_INLINE error_codes unescape(internal::read_structural_index_ptr stringIter, value_type& result) const noexcept { + internal::read_buffer_ptr stringStart = doc->root + *stringIter + 1; + if (const uint64_t directLength = zeroCopyLength(stringStart); directLength != noZeroCopy) [[likely]] { + result = value_type{ std::bit_cast(stringStart), static_cast(directLength) }; + return error_codes::success; + } + char* scratch = doc->arena->scratch(); + const auto parsed = internal::string_parser::impl(stringStart, doc->end, scratch); + if (parsed.outLength == std::numeric_limits::max()) [[unlikely]] { + return error_codes::string_error; + } + if (parsed.outLength == parsed.rawLength) { + result = value_type{ std::bit_cast(stringStart), static_cast(parsed.rawLength) }; + return error_codes::success; + } + char* slot = doc->arena->slot(static_cast(stringStart - doc->root)); + internal::jsonifierMemcpy(slot, scratch, parsed.outLength); + result = value_type{ slot, static_cast(parsed.outLength) }; + return error_codes::success; + } + + JSONIFIER_INLINE static string_stop_masks stringStopMasks(internal::read_buffer_ptr current) noexcept { + const internal::jsonifier_simd_int_t values = internal::simd::gatherValuesU(current); + const internal::jsonifier_simd_int_t backslashes = internal::simd::gatherValue('\\'); + const internal::jsonifier_simd_int_t quotes = internal::simd::gatherValue('"'); + const internal::jsonifier_simd_int_t controlCeiling = internal::simd::gatherValue(static_cast(0x20)); +#if JSONIFIER_AVX || JSONIFIER_AVX2 || JSONIFIER_AVX512 + using mask_type = std::make_unsigned_t; + const uint64_t nonAscii = static_cast(static_cast(internal::simd::opBitMask(values))); + const auto stopsAndHigh = internal::simd::opOr(internal::simd::opOr(internal::simd::opCmpEqRaw(values, backslashes), internal::simd::opCmpEqRaw(values, quotes)), + internal::simd::opCmpLtSignedRaw(values, controlCeiling)); + return { nonAscii, static_cast(static_cast(internal::simd::opBitMask(stopsAndHigh))) & ~nonAscii }; +#else + const internal::jsonifier_simd_int_t asciiCeiling = internal::simd::gatherValue(static_cast(0x7F)); + const auto stops = internal::simd::opOr(internal::simd::opOr(internal::simd::opCmpEqRaw(values, backslashes), internal::simd::opCmpEqRaw(values, quotes)), + internal::simd::opCmpLtRaw(values, controlCeiling)); + #if JSONIFIER_NEON + return { internal::simd::opBitMaskRaw(internal::simd::opCmpLtRaw(asciiCeiling, values)), internal::simd::opBitMaskRaw(stops) }; + #else + using mask_type = std::make_unsigned_t; + return { static_cast(static_cast(internal::simd::opBitMask(internal::simd::opCmpLtRaw(asciiCeiling, values)))), + static_cast(static_cast(internal::simd::opBitMask(stops))) }; + #endif +#endif + } + + JSONIFIER_INLINE static uint64_t resolveMaskedStop(internal::read_buffer_ptr stringStart, internal::read_buffer_ptr current, string_stop_masks masks, + bool sawNonAscii) noexcept { + const uint64_t beforeStop = (masks.stops & (0 - masks.stops)) - 1; + return resolveStringStop(stringStart, current + (internal::simd::countrZeroUnsafe(masks.stops) >> stopMaskLaneShift), + sawNonAscii || (masks.nonAscii & beforeStop) != 0); + } + + JSONIFIER_INLINE static uint64_t resolveStringStop(internal::read_buffer_ptr stringStart, internal::read_buffer_ptr stop, bool sawNonAscii) noexcept { + if (*stop != internal::uint_c_v<'"'>) [[unlikely]] { + return noZeroCopy; + } + const uint64_t length = static_cast(stop - stringStart); + return !sawNonAscii || validUtf8(stringStart, length) ? length : noZeroCopy; + } + + JSONIFIER_INLINE static error_codes finishNumber(internal::read_buffer_ptr parsedEnd, internal::read_buffer_ptr valueEnd) noexcept { + if (parsedEnd && (parsedEnd == valueEnd || internal::validPostPrimitiveTable[static_cast(*parsedEnd)])) [[likely]] { + return error_codes::success; + } + return error_codes::number_error; + } + + template JSONIFIER_INLINE bool atomMatches() const noexcept { + internal::read_buffer_ptr valueStart = doc->root + *iter; + if (static_cast(doc->end - valueStart) < atomSize) [[unlikely]] { + return false; + } + uint32_t comparison; + internal::pow2MemcpyWrapper<4>(&comparison, valueStart); + if constexpr (std::endian::native == std::endian::big) { + comparison = internal::byteswap(comparison); + } + if constexpr (atomSize == 5) { + if (comparison != leadingWord || valueStart[4] != internal::uint_c_v<'e'>) [[unlikely]] { + return false; + } + } else { + if (comparison != leadingWord) [[unlikely]] { + return false; + } + } + return valueStart + atomSize == doc->end || internal::validPostPrimitiveTable[static_cast(valueStart[atomSize])]; + } + + template JSONIFIER_INLINE uint64_t zeroCopyLength(internal::read_buffer_ptr stringStart) const noexcept { + internal::read_buffer_ptr current = stringStart; + bool sawNonAscii{}; + if (static_cast(doc->end - current) >= internal::simdBytesPerRegister) [[likely]] { + if (const string_stop_masks masks = selectStops(stringStopMasks(current)); masks.stops != 0) [[likely]] { + return resolveMaskedStop(stringStart, current, masks, false); + } else { + sawNonAscii = masks.nonAscii != 0; + } + current += internal::simdBytesPerRegister; + while (static_cast(doc->end - current) >= internal::simdBytesPerRegister) { + if (const string_stop_masks masks = selectStops(stringStopMasks(current)); masks.stops != 0) { + return resolveMaskedStop(stringStart, current, masks, sawNonAscii); + } else { + sawNonAscii = sawNonAscii || masks.nonAscii != 0; + } + current += internal::simdBytesPerRegister; + } + } + uint8_t accumulated{ static_cast(sawNonAscii ? 0x80 : 0x00) }; + for (; current < doc->end; ++current) { + const uint8_t currentChar = static_cast(*current); + if ((allowNonAscii ? internal::utf8StringStopTable : internal::asciiStringStopTable)[currentChar]) { + return resolveStringStop(stringStart, current, (accumulated & 0x80) != 0); + } + accumulated |= currentChar; + } + return noZeroCopy; + } + + template JSONIFIER_INLINE static string_stop_masks selectStops(string_stop_masks masks) noexcept { + if constexpr (!allowNonAscii) { + masks.stops |= masks.nonAscii; + masks.nonAscii = 0; + } + return masks; + } + + JSONIFIER_INLINE static bool unescapePointerToken(string_view& token, string& unescapedToken) noexcept; + + JSONIFIER_INLINE static error_codes parsePointerIndex(string_view token, uint64_t& index) noexcept; + + JSONIFIER_INLINE value pointerStep(string_view token, string& unescapedToken) const noexcept; + + JSONIFIER_INLINE bool isNumberStart() const noexcept { + return error_code == error_codes::success && internal::typeTable[doc->charAt(iter)] == json_types::number; + } + + JSONIFIER_INLINE bool falseMatches() const noexcept { + static constexpr uint32_t falseVal{ 0b01110011'01101100'01100001'01100110 }; + return atomMatches(); + } + + JSONIFIER_INLINE bool trueMatches() const noexcept { + static constexpr uint32_t trueVal{ 0b01100101'01110101'01110010'01110100 }; + return atomMatches(); + } + + JSONIFIER_INLINE bool nullMatches() const noexcept { + static constexpr uint32_t nullVal{ 0b01101100'01101100'01110101'01101110 }; + return atomMatches(); + } + + JSONIFIER_INLINE error_codes typeError() const noexcept { + return error_code != error_codes::success ? error_code : error_codes::incorrect_type; + } + + JSONIFIER_INLINE static bool isWhitespace(uint8_t c) noexcept { + return internal::whitespaceTable[c]; + } + + JSONIFIER_INLINE void consume() const noexcept { + if (doc->cursor == iter) [[likely]] { + doc->cursor = iter + 1; + } + } + + internal::read_structural_index_ptr iter{}; + document_state* doc{}; + uint64_t depth{}; + error_codes error_code{ error_codes::success }; + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/parsing/string_parser.hpp b/deps/jsonifier/include/jsonifier-incl/parsing/string_parser.hpp new file mode 100644 index 000000000000..ccc5b23b5574 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/parsing/string_parser.hpp @@ -0,0 +1,154 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/parsing/string_parser.hpp + */ +#pragma once + +#include + +namespace jsonifier::internal { + + // Sampled from Dr. Lemire's library, simdjson: https://github.com/simdjson/simdjson + struct string_parser { + using scan_result = typename string_scanner::scan_result; + + struct parse_state { + internal::utf8_validation_state validationState{}; + scan_result result{}; + }; + + struct parse_step { + template JSONIFIER_INLINE static basic_iterator01 impl(basic_iterator01& current, + const basic_iterator01 end, basic_iterator02& out, const basic_iterator01 stringStart, const basic_iterator02 outStart, parse_state& state) noexcept { + using simd_list_local = type_list_element_t; + using integer_type = typename simd_list_local::integer_type; + static constexpr auto avx_type = simd_list_local::type::avx_type; + using simd_type_local = typename simd_type_wrapper::type; + static constexpr uint64_t bytesProcessed = simd_list_local::bytesProcessed; + internal::utf8_register_validator> validator{ state.validationState }; + const simd_type_local backslashes = simd::gatherValue('\\'); + const simd_type_local quotes = simd::gatherValue('"'); + const simd_type_local controls = simd::gatherValue(static_cast(32)); + while (static_cast(end - current) >= bytesProcessed) { + const simd_type_local values = simd::gatherValuesU(current); + simd::storeU(values, out); + const integer_type stops = static_cast( + simd::opBitMask(simd::opOr(simd::opOr(simd::opCmpEqRaw(values, backslashes), simd::opCmpEqRaw(values, quotes)), simd::opCmpLtRaw(values, controls)))); + if (stops == static_cast(0)) { + validator.checkRegister(values); + current += bytesProcessed; + out += bytesProcessed; + continue; + } + if (!resolveStop(validator, static_cast(simd::countrZeroUnsafe(stops)), current, end, out, stringStart, outStart, state)) { + return nullptr; + } + } + validator.flush(); + if constexpr (index == listSize - 1) { + while (current < end) { + const uint64_t remaining = static_cast(end - current); + alignas(simdBytesPerRegister) uint8_t padded[bytesProcessed]; + std::memset(padded, 32, bytesProcessed); + jsonifierMemcpyUpTo(padded, current, remaining); + const simd_type_local values = simd::gatherValues(std::bit_cast(+padded)); + const integer_type stops = static_cast( + simd::opBitMask(simd::opOr(simd::opOr(simd::opCmpEqRaw(values, backslashes), simd::opCmpEqRaw(values, quotes)), simd::opCmpLtRaw(values, controls)))); + if (stops == static_cast(0)) { + return nullptr; + } + const uint64_t offset = static_cast(simd::countrZeroUnsafe(stops)); + jsonifierMemcpyUpTo(out, current, offset); + if (!resolveStop(validator, offset, current, end, out, stringStart, outStart, state)) { + return nullptr; + } + } + return nullptr; + } else { + return current; + } + } + }; + + template JSONIFIER_INLINE static bool resolveStop(validator_type& validator, uint64_t offset, + basic_iterator01& current, const basic_iterator01 end, basic_iterator02& out, const basic_iterator01 stringStart, const basic_iterator02 outStart, + parse_state& state) noexcept { + const uint8_t foundChar = static_cast(current[offset]); + if (foundChar < 32) [[unlikely]] { + return false; + } + validator.checkPartial(current, offset); + current += offset; + out += offset; + if (validator.errors()) [[unlikely]] { + return false; + } + if (foundChar == uint_c_v<'"'>) { + state.result = { static_cast(out - outStart), static_cast(current - stringStart) }; + return false; + } + if (!parseEscape(current, end, out)) [[unlikely]] { + return false; + } + validator.reset(); + return true; + } + + template + JSONIFIER_INLINE static scan_result impl(basic_iterator01 current, const basic_iterator01 end, basic_iterator02 out) noexcept { + const auto stringStart = current; + const auto outStart = out; + parse_state state{}; + string_parse_executor>::impl(current, end, out, stringStart, outStart, state); + return state.result; + } + + template + JSONIFIER_INLINE static bool parseEscape(basic_iterator01& current, const basic_iterator01 end, basic_iterator02& out) noexcept { + if (end - current < 2) [[unlikely]] { + return false; + } + if (current[1] == 'u') { + do { + if (!handleUnicodeCodePoint(current, out, end)) [[unlikely]] { + return false; + } + } while (end - current >= 2 && current[0] == '\\' && current[1] == 'u'); + return true; + } + const uint8_t decoded = static_cast(escapeMap[static_cast(current[1])]); + if (decoded == 0u) [[unlikely]] { + return false; + } + *out = static_cast>(decoded); + ++out; + current += 2; + return true; + } + }; + + struct string_parse_context { + string_parse_context() noexcept = delete; + + JSONIFIER_INLINE string_parse_context(read_buffer_ptr sourceNew, read_buffer_ptr sourceEndNew, uint64_t& outLengthNew) noexcept + : sourceEnd{ sourceEndNew }, source{ sourceNew }, outLength{ outLengthNew } { + } + + string_parse_context& operator=(string_parse_context&&) noexcept = delete; + string_parse_context(string_parse_context&&) noexcept = delete; + string_parse_context& operator=(const string_parse_context&) noexcept = delete; + string_parse_context(const string_parse_context&) noexcept = delete; + + JSONIFIER_INLINE uint64_t operator()(char* __restrict ptrNew, uint64_t) const noexcept { + outLength = string_parser::impl(source, sourceEnd, static_cast(ptrNew)).outLength; + return outLength == std::numeric_limits::max() ? uint64_t{} : outLength; + } + + read_buffer_ptr sourceEnd{}; + read_buffer_ptr source{}; + uint64_t& outLength; + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/simd/add_tape_values.hpp b/deps/jsonifier/include/jsonifier-incl/simd/add_tape_values.hpp new file mode 100644 index 000000000000..e1f0c3ee5948 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/simd/add_tape_values.hpp @@ -0,0 +1,116 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/simd/add_tape_values.hpp + */ +#pragma once + +#include +#include +#include +#include + +namespace jsonifier::internal { + + template struct add_tape_values; + +#if JSONIFIER_AVX512 + + // Sampled from Dr. Lemire's library, simdjson: https://github.com/simdjson/simdjson + template struct add_tape_values> { + using size_type = uint64_t; + static constexpr uint64_t blocksPerStep{ sizeof...(indices) }; + + JSONIFIER_INLINE static void impl(const array& __restrict bitsArr, const array& __restrict cnts, + write_structural_index_ptr tape, size_type strIdx) noexcept { + uint64_t offset = 0; + const __m512i compressionMask{ _mm512_set_epi32(0x3f3e3d3c, 0x3b3a3938, 0x37363534, 0x33323130, 0x2f2e2d2c, 0x2b2a2928, 0x27262524, 0x23222120, 0x1f1e1d1c, 0x1b1a1918, + 0x17161514, 0x13121110, 0x0f0e0d0c, 0x0b0a0908, 0x07060504, 0x03020100) }; + const __m512i strIdxVec{ _mm512_set1_epi32(static_cast(strIdx)) }; + (((drainLane(compressionMask, strIdxVec, bitsArr, cnts, tape + offset)), offset += cnts[tag{}]), ...); + } + + template JSONIFIER_INLINE static void drainLane(const __m512i compressionMask, const __m512i strIdxVec, + const array& __restrict bitsArr, const array& __restrict cnts, write_structural_index_ptr tape) noexcept { + uint64_t bits = bitsArr[tag{}]; + if (!static_cast(bits)) [[unlikely]] { + return; + } + const uint64_t count = cnts[tag{}]; + static constexpr int32_t bitTotal{ static_cast(index * 64ull) }; + const __m512i indexes = _mm512_maskz_compress_epi8(bits, compressionMask); + const __m512i startIndexLocal = _mm512_add_epi32(strIdxVec, _mm512_set1_epi32(bitTotal)); + __m512i t0 = _mm512_cvtepu8_epi32(_mm512_castsi512_si128(indexes)); + _mm512_storeu_si512(tape, _mm512_add_epi32(t0, startIndexLocal)); + if (count > 16) { + const __m512i t1 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 1)); + _mm512_storeu_si512(tape + 16, _mm512_add_epi32(t1, startIndexLocal)); + if (count > 32) { + const __m512i t2 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 2)); + _mm512_storeu_si512(tape + 32, _mm512_add_epi32(t2, startIndexLocal)); + if (count > 48) { + const __m512i t3 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 3)); + _mm512_storeu_si512(tape + 48, _mm512_add_epi32(t3, startIndexLocal)); + } + } + } + } + }; + +#else + + template struct write_indices_functor { + using size_type = uint64_t; + + template JSONIFIER_INLINE static void impl(size_type base, size_type& __restrict bits, write_structural_index_ptr tape) noexcept { + { + tape[static_cast(tag{})] = simd::tape_writer_op::extractIndex(base, bits); + bits = simd::tape_writer_op::advance(bits); + } + } + }; + + template struct write_indices_stepped_functor { + using size_type = uint64_t; + + template JSONIFIER_INLINE static bool impl(size_type base, size_type& __restrict bits, write_structural_index_ptr tape, uint64_t cnt) noexcept { + if constexpr (index > 0) { + if ((index < cnt)) [[unlikely]] { + { + functor_runner>::impl(base, bits, tape + index); + } + return true; + } else { + return false; + } + } else { + functor_runner>::impl(base, bits, tape + index); + return true; + } + } + }; + + template struct add_tape_values> { + using size_type = uint64_t; + static constexpr uint64_t blocksPerStep{ sizeof...(indices) }; + + JSONIFIER_INLINE static void impl(const array& __restrict bitsArr, const array& __restrict cnts, + write_structural_index_ptr tape, size_type strIdx) noexcept { + uint64_t offset = 0; + (((drainLane(bitsArr, cnts, tape + offset, strIdx)), offset += cnts[tag{}]), ...); + } + + template JSONIFIER_INLINE static void drainLane(const array& __restrict bitsArr, + const array& __restrict cnts, write_structural_index_ptr tape, size_type strIdx) noexcept { + uint64_t bits = bitsArr[tag{}]; + const uint64_t cnt = cnts[tag{}]; + static constexpr size_type bitTotal{ tag{} * 64ull }; + const size_type base = bitTotal + strIdx; + functor_runner, simdTapeStep>::implAnd(base, bits, tape, cnt); + } + }; + +#endif + +} diff --git a/deps/jsonifier/include/jsonifier-incl/simd/avx.hpp b/deps/jsonifier/include/jsonifier-incl/simd/avx.hpp new file mode 100644 index 000000000000..ed1d6f32883f --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/simd/avx.hpp @@ -0,0 +1,328 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/simd/avx.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::internal::simd { + +#if JSONIFIER_AVX512 || JSONIFIER_AVX2 || JSONIFIER_AVX + + template [[maybe_unused]] JSONIFIER_INLINE auto opCmpEqRaw(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm_cmpeq_epi8(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opCmpLtRaw(simd_int_t01 value, simd_int_t02 other) noexcept { + const jsonifier_simd_int_128 offset = _mm_set1_epi8(static_cast(0x80)); + return _mm_cmpgt_epi8(_mm_add_epi8(other, offset), _mm_add_epi8(value, offset)); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opShuffle(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm_shuffle_epi8(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opCmpEq(simd_int_t01 value, simd_int_t02 other) noexcept { + return static_cast(_mm_movemask_epi8(_mm_cmpeq_epi8(value, other))); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opSubs(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm_subs_epu8(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opXor(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm_xor_si128(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opAnd(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm_and_si128(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opOr(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm_or_si128(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE void storeU(simd_int_type_new value, void* __restrict storageLocation) noexcept { + _mm_storeu_si128(static_cast<__m128i*>(storageLocation), value); + } + + template [[maybe_unused]] JSONIFIER_INLINE void store(simd_int_type_new value, void* __restrict storageLocation) noexcept { + _mm_store_si128(static_cast<__m128i*>(storageLocation), value); + } + + template [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValuesU(const void* __restrict str) noexcept { + return _mm_loadu_si128(static_cast(str)); + } + + template [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValues(const void* __restrict str) noexcept { + return _mm_load_si128(static_cast(str)); + } + + template JSONIFIER_INLINE simd_int_t01 orAll(const simd_array chunks) noexcept { + return opOr(chunks.template get<0>(), opOr(chunks.template get<1>(), opOr(chunks.template get<2>(), chunks.template get<3>()))); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opSrLi(simd_int_t01 value) noexcept { + return _mm_srli_epi16(value, alignment); + } + + template [[maybe_unused]] JSONIFIER_INLINE uint64_t postCmpTzcntUnsafe(const value_type value) noexcept { + return countrZeroUnsafe(value); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opBitMaskRaw(simd_int_t01 value) noexcept { + return _mm_movemask_epi8(value); + } + + template [[maybe_unused]] JSONIFIER_INLINE uint64_t postCmpTzcnt(const value_type value) noexcept { + return countrZero(value); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opBitMask(simd_int_t01 value) noexcept { + return _mm_movemask_epi8(value); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opTest(simd_int_t01 value) noexcept { + return _mm_testz_si128(value, value); + } + + template + [[maybe_unused]] JSONIFIER_INLINE auto opCmpLtSignedRaw(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm_cmpgt_epi8(other, value); + } + + template + [[maybe_unused]] JSONIFIER_INLINE auto opAlignR(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm_alignr_epi8(value, other, alignment); + } + + template + [[maybe_unused]] JSONIFIER_INLINE auto opPrev(simd_int_t01 current, simd_int_t02 previous) noexcept { + return opAlignR(current, previous); + } + + template + requires(sizeof(char_t) == 8) + [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValue(const char_t str) noexcept { + return _mm_set1_epi64x(static_cast(str)); + } + + template + requires(sizeof(char_t) == 1) + [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValue(const char_t str) noexcept { + return _mm_set1_epi8(static_cast(str)); + } + + #if JSONIFIER_AVX512 || JSONIFIER_AVX2 + + template [[maybe_unused]] JSONIFIER_INLINE auto opCmpEqRaw(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm256_cmpeq_epi8(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opCmpLtRaw(simd_int_t01 value, simd_int_t02 other) noexcept { + const jsonifier_simd_int_256 offset = _mm256_set1_epi8(static_cast(0x80)); + return _mm256_cmpgt_epi8(_mm256_add_epi8(other, offset), _mm256_add_epi8(value, offset)); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opShuffle(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm256_shuffle_epi8(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opCmpEq(simd_int_t01 value, simd_int_t02 other) noexcept { + return static_cast(_mm256_movemask_epi8(_mm256_cmpeq_epi8(value, other))); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opSubs(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm256_subs_epu8(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opXor(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm256_xor_si256(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opAnd(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm256_and_si256(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opOr(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm256_or_si256(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE void storeU(simd_int_type_new value, void* __restrict storageLocation) noexcept { + _mm256_storeu_si256(static_cast<__m256i*>(storageLocation), value); + } + + template [[maybe_unused]] JSONIFIER_INLINE void store(simd_int_type_new value, void* __restrict storageLocation) noexcept { + _mm256_store_si256(static_cast<__m256i*>(storageLocation), value); + } + + template [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValuesU(const void* __restrict str) noexcept { + return _mm256_loadu_si256(static_cast(str)); + } + + template [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValues(const void* __restrict str) noexcept { + return _mm256_load_si256(static_cast(str)); + } + + template JSONIFIER_INLINE simd_int_t01 orAll(const simd_array chunks) noexcept { + return opOr(chunks.template get<0>(), chunks.template get<1>()); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opSrLi(simd_int_t01 value) noexcept { + return _mm256_srli_epi16(value, alignment); + } + + template JSONIFIER_INLINE simd_type opPrev(simd_type cur, simd_type prev) noexcept { + return _mm256_alignr_epi8(cur, _mm256_permute2x128_si256(prev, cur, 0x21), shift); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opBitMaskRaw(simd_int_t01 value) noexcept { + return _mm256_movemask_epi8(value); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opBitMask(simd_int_t01 value) noexcept { + return _mm256_movemask_epi8(value); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opTest(simd_int_t01 value) noexcept { + return _mm256_testz_si256(value, value); + } + + template + [[maybe_unused]] JSONIFIER_INLINE auto opCmpLtSignedRaw(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm256_cmpgt_epi8(other, value); + } + + template + [[maybe_unused]] JSONIFIER_INLINE auto opAlignR(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm256_alignr_epi8(value, other, alignment); + } + + template + requires(sizeof(char_t) == 8) + [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValue(const char_t value) noexcept { + return _mm256_set1_epi64x(static_cast(value)); + } + + template + requires(sizeof(char_t) == 1) + [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValue(const char_t value) noexcept { + return _mm256_set1_epi8(static_cast(value)); + } + + #if JSONIFIER_AVX512 + + template [[maybe_unused]] JSONIFIER_INLINE auto opCmpEqRaw(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm512_movm_epi8(_mm512_cmpeq_epi8_mask(value, other)); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opCmpLtRaw(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm512_movm_epi8(_mm512_cmplt_epu8_mask(value, other)); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opShuffle(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm512_shuffle_epi8(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opCmpNeq(simd_int_t01 value, simd_int_t02 other) noexcept { + return static_cast(_mm512_cmpneq_epi8_mask(value, other)); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opCmpEq(simd_int_t01 value, simd_int_t02 other) noexcept { + return static_cast(_mm512_cmpeq_epi8_mask(value, other)); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opSubs(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm512_subs_epu8(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opXor(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm512_xor_si512(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opAnd(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm512_and_si512(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opOr(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm512_or_si512(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE void storeU(simd_int_type_new value, void* __restrict storageLocation) noexcept { + _mm512_storeu_si512(static_cast<__m512i*>(storageLocation), value); + } + + template [[maybe_unused]] JSONIFIER_INLINE void store(simd_int_type_new value, void* __restrict storageLocation) noexcept { + _mm512_store_si512(static_cast<__m512i*>(storageLocation), value); + } + + template [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValuesU(const void* __restrict str) noexcept { + return _mm512_loadu_si512(static_cast(str)); + } + + template [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValues(const void* __restrict str) noexcept { + return _mm512_load_si512(static_cast(str)); + } + + template JSONIFIER_INLINE simd_int_t01 orAll(const simd_array chunks) noexcept { + return chunks.template get<0>(); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opSrLi(simd_int_t01 value) noexcept { + return _mm512_srli_epi16(value, alignment); + } + + template + [[maybe_unused]] JSONIFIER_INLINE auto opPrev(simd_int_t01 current, simd_int_t02 previous) noexcept { + return _mm512_alignr_epi8(current, _mm512_permutex2var_epi64(previous, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), current), shift); + } + + template [[maybe_unused]] JSONIFIER_INLINE uint64_t opBitMask(simd_int_t01 value) noexcept { + return static_cast(_mm512_movepi8_mask(value)); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opBitMaskRaw(simd_int_t01 value) noexcept { + return _mm512_movepi8_mask(value); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opTest(simd_int_t01 value) noexcept { + return _mm512_test_epi64_mask(value, value) == 0; + } + + template + [[maybe_unused]] JSONIFIER_INLINE auto opCmpLtSignedRaw(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm512_movm_epi8(_mm512_cmplt_epi8_mask(value, other)); + } + + template + [[maybe_unused]] JSONIFIER_INLINE auto opAlignR(simd_int_t01 value, simd_int_t02 other) noexcept { + return _mm512_alignr_epi8(value, other, alignment); + } + + template + requires(sizeof(char_t) == 8) + [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValue(const char_t value) noexcept { + return _mm512_set1_epi64(static_cast(value)); + } + + template + requires(sizeof(char_t) == 1) + [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValue(const char_t value) noexcept { + return _mm512_set1_epi8(static_cast(value)); + } + + #endif + + #endif + + template JSONIFIER_INLINE bool isAscii(simd_type value) noexcept { + return opBitMask(value) == 0; + } + +#endif + +} diff --git a/deps/jsonifier/include/jsonifier-incl/simd/avx_stage1.hpp b/deps/jsonifier/include/jsonifier-incl/simd/avx_stage1.hpp new file mode 100644 index 000000000000..9f7d20e308ae --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/simd/avx_stage1.hpp @@ -0,0 +1,320 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/simd/avx_stage1.hpp + */ +// Sampled from Dr. Lemire's library, simdjson: https://github.com/simdjson/simdjson +#pragma once + +#include +#include +#include + +namespace jsonifier::internal::simd { + +#if !JSONIFIER_NEON && !JSONIFIER_SVE2 + + inline constexpr array shiftAmounts{ [] { + array returnValue{}; + for (uint64_t x = 0; x < simdRegistersPerBlock; ++x) { + returnValue[x] = simdBytesPerRegister * x; + } + return returnValue; + }() }; + + consteval uint64_t getShiftAmount(const uint64_t index) noexcept { + return shiftAmounts[index % shiftAmounts.size()]; + } + + struct prefix_xor_op { + JSONIFIER_INLINE static uint64_t impl(uint64_t bitmask) noexcept { + #if JSONIFIER_CLMUL + const __m128i allOnes = _mm_set1_epi8(static_cast(-1)); + const __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0, static_cast(bitmask)), allOnes, 0); + return static_cast(_mm_cvtsi128_si64(result)); + #else + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; + #endif + } + }; + + struct cmp_eq_op { + JSONIFIER_INLINE static uint64_t impl(const simd_array_t lhs, const jsonifier_simd_int_t rhsBroadcast) noexcept { + uint64_t result = opCmpEq(lhs.get<0>(), rhsBroadcast); + if constexpr (simdRegistersPerBlock > 1) { + result |= static_cast(opCmpEq(lhs.get<1>(), rhsBroadcast)) << getShiftAmount(1); + if constexpr (simdRegistersPerBlock > 2) { + result |= static_cast(opCmpEq(lhs.get<2>(), rhsBroadcast)) << getShiftAmount(2); + result |= static_cast(opCmpEq(lhs.get<3>(), rhsBroadcast)) << getShiftAmount(3); + } + } + return result; + } + + JSONIFIER_INLINE static uint64_t impl(const simd_array_t lhs, const simd_array_t rhs) noexcept { + uint64_t result = opCmpEq(lhs.get<0>(), rhs.get<0>()); + if constexpr (simdRegistersPerBlock > 1) { + result |= static_cast(opCmpEq(lhs.get<1>(), rhs.get<1>())) << getShiftAmount(1); + if constexpr (simdRegistersPerBlock > 2) { + result |= static_cast(opCmpEq(lhs.get<2>(), rhs.get<2>())) << getShiftAmount(2); + result |= static_cast(opCmpEq(lhs.get<3>(), rhs.get<3>())) << getShiftAmount(3); + } + } + return result; + } + }; + + struct unescaped_collector { + JSONIFIER_INLINE static uint64_t impl(const simd_array_t ltRhs) noexcept { + uint64_t result = static_cast(opBitMask(ltRhs.get<0>())); + if constexpr (simdRegistersPerBlock > 1) { + result |= static_cast(opBitMask(ltRhs.get<1>())) << getShiftAmount(1); + if constexpr (simdRegistersPerBlock > 2) { + result |= static_cast(opBitMask(ltRhs.get<2>())) << getShiftAmount(2); + result |= static_cast(opBitMask(ltRhs.get<3>())) << getShiftAmount(3); + } + } + return result; + } + }; + + struct bracket_masks { + uint64_t closes; + uint64_t opens; + }; + + struct bracket_collector { + JSONIFIER_INLINE static bracket_masks impl(const simd_array_t in01, const jsonifier_simd_int_t spaceMask, const jsonifier_simd_int_t openRegister, + const jsonifier_simd_int_t closeRegister) noexcept { + simd_array_t folded; + folded.set<0>(opOr(in01.get<0>(), spaceMask)); + if constexpr (simdRegistersPerBlock > 1) { + folded.set<1>(opOr(in01.get<1>(), spaceMask)); + if constexpr (simdRegistersPerBlock > 2) { + folded.set<2>(opOr(in01.get<2>(), spaceMask)); + folded.set<3>(opOr(in01.get<3>(), spaceMask)); + } + } + return { cmp_eq_op::impl(folded, closeRegister), cmp_eq_op::impl(folded, openRegister) }; + } + }; + + template struct rope_detector : rope_block { + template JSONIFIER_INLINE void nextScalar(const pod_simd_array_t in01, + const simd_register_t bsRegister, const simd_register_t quoteRegister) noexcept; + + JSONIFIER_INLINE void next(const simd_array_t in01, const jsonifier_simd_int_t bsRegister, const jsonifier_simd_int_t quoteRegister) noexcept { + const uint64_t escaped = nextEscapeAndTerminalCode(cmp_eq_op::impl(in01, bsRegister)); + const uint64_t quotes = (cmp_eq_op::impl(in01, quoteRegister) & ~escaped); + rope_block::escaped = escaped; + rope_block::quotes = quotes; + return quotes ? finishNext() : finishNextNoInString(); + } + + JSONIFIER_INLINE uint64_t nextEscapeAndTerminalCode(const uint64_t backslashLocal) noexcept { + if (!backslashLocal) { + const uint64_t escaped = nextIsEscaped; + nextIsEscaped = 0; + return escaped; + } + const uint64_t escapeAndTerminalCode = nextEscapeAndTerminalCodeImpl(backslashLocal & ~nextIsEscaped); + const uint64_t escaped = escapeAndTerminalCode ^ (backslashLocal | nextIsEscaped); + nextIsEscaped = (escapeAndTerminalCode & backslashLocal) >> 63; + return escaped; + } + + JSONIFIER_INLINE uint64_t nextEscapeAndTerminalCodeImpl(const uint64_t potentialEscape) noexcept { + static constexpr uint64_t oddBits{ 0xAAAAAAAAAAAAAAAAULL }; + const uint64_t maybeEscaped = potentialEscape << 1; + const uint64_t maybeEscapedAndOddBits = maybeEscaped | oddBits; + const uint64_t evenSeriesCodesAndOddBits = maybeEscapedAndOddBits - potentialEscape; + return evenSeriesCodesAndOddBits ^ oddBits; + } + + JSONIFIER_INLINE uint64_t followsNonquoteScalar(const uint64_t nonquoteScalar) noexcept { + const uint64_t shifted = (nonquoteScalar << 1) | prevScalar; + prevScalar = nonquoteScalar >> 63; + return shifted; + } + + JSONIFIER_INLINE void finishNext() noexcept { + const uint64_t inString = prefix_xor_op::impl(rope_block::quotes) ^ prevInString; + prevInString = static_cast(static_cast(inString) >> 63); + rope_block::inString = inString; + } + + JSONIFIER_INLINE void finishNextNoInString() noexcept { + rope_block::inString = prevInString; + } + + uint64_t nextIsEscaped{}; + uint64_t prevInString{}; + uint64_t prevScalar{}; + }; + + struct ws_collector { + JSONIFIER_INLINE static uint64_t impl(const simd_array_t in01, const jsonifier_simd_int_t whitespaceTableLocal) noexcept { + simd_array_t wsShuffle; + wsShuffle.set<0>(opShuffle(whitespaceTableLocal, in01.get<0>())); + if constexpr (simdRegistersPerBlock > 1) { + wsShuffle.set<1>(opShuffle(whitespaceTableLocal, in01.get<1>())); + if constexpr (simdRegistersPerBlock > 2) { + wsShuffle.set<2>(opShuffle(whitespaceTableLocal, in01.get<2>())); + wsShuffle.set<3>(opShuffle(whitespaceTableLocal, in01.get<3>())); + } + } + return cmp_eq_op::impl(in01, wsShuffle); + } + }; + + struct op_collector { + JSONIFIER_INLINE static uint64_t impl(const simd_array_t in01, const jsonifier_simd_int_t opTable, const jsonifier_simd_int_t spaceMask) noexcept { + simd_array_t orLhs; + simd_array_t shuffleRhs; + + orLhs.set<0>(opOr(in01.get<0>(), spaceMask)); + shuffleRhs.set<0>(opShuffle(opTable, in01.get<0>())); + if constexpr (simdRegistersPerBlock > 1) { + orLhs.set<1>(opOr(in01.get<1>(), spaceMask)); + shuffleRhs.set<1>(opShuffle(opTable, in01.get<1>())); + if constexpr (simdRegistersPerBlock > 2) { + orLhs.set<2>(opOr(in01.get<2>(), spaceMask)); + shuffleRhs.set<2>(opShuffle(opTable, in01.get<2>())); + orLhs.set<3>(opOr(in01.get<3>(), spaceMask)); + shuffleRhs.set<3>(opShuffle(opTable, in01.get<3>())); + } + } + return cmp_eq_op::impl(orLhs, shuffleRhs); + } + }; + + template struct pod_cmp_eq_op { + using simd_type = simd_register_t; + using simd_array_type = pod_simd_array_t; + + JSONIFIER_INLINE static uint64_t impl(const simd_array_type lhs, const simd_array_type rhs) noexcept { + uint64_t result = static_cast(opCmpEq(lhs.template get<0>(), rhs.template get<0>())); + if constexpr (registerCount > 1) { + result |= static_cast(opCmpEq(lhs.template get<1>(), rhs.template get<1>())) << (registerBytes * 1); + if constexpr (registerCount > 2) { + result |= static_cast(opCmpEq(lhs.template get<2>(), rhs.template get<2>())) << (registerBytes * 2); + result |= static_cast(opCmpEq(lhs.template get<3>(), rhs.template get<3>())) << (registerBytes * 3); + } + } + return result; + } + + JSONIFIER_INLINE static uint64_t impl(const simd_array_type lhs, const simd_type rhsBroadcast) noexcept { + uint64_t result = static_cast(opCmpEq(lhs.template get<0>(), rhsBroadcast)); + if constexpr (registerCount > 1) { + result |= static_cast(opCmpEq(lhs.template get<1>(), rhsBroadcast)) << (registerBytes * 1); + if constexpr (registerCount > 2) { + result |= static_cast(opCmpEq(lhs.template get<2>(), rhsBroadcast)) << (registerBytes * 2); + result |= static_cast(opCmpEq(lhs.template get<3>(), rhsBroadcast)) << (registerBytes * 3); + } + } + return result; + } + + static_assert(registerCount >= 1 && registerCount * registerBytes <= simdBytesPerBlock); + }; + + template struct pod_ws_collector { + using simd_type = simd_register_t; + using simd_array_type = pod_simd_array_t; + + JSONIFIER_INLINE static uint64_t impl(const simd_array_type in01, const simd_type whitespaceTableLocal) noexcept { + simd_array_type wsShuffle; + wsShuffle.template set<0>(opShuffle(whitespaceTableLocal, in01.template get<0>())); + if constexpr (registerCount > 1) { + wsShuffle.template set<1>(opShuffle(whitespaceTableLocal, in01.template get<1>())); + if constexpr (registerCount > 2) { + wsShuffle.template set<2>(opShuffle(whitespaceTableLocal, in01.template get<2>())); + wsShuffle.template set<3>(opShuffle(whitespaceTableLocal, in01.template get<3>())); + } + } + return pod_cmp_eq_op::impl(in01, wsShuffle); + } + }; + + template struct scalar_op_collector { + using simd_type = simd_register_t; + using simd_array_type = pod_simd_array_t; + + JSONIFIER_INLINE static uint64_t impl(const simd_array_type in01, const simd_type opTable, const simd_type spaceMask) noexcept { + simd_array_type orLhs; + simd_array_type shuffleRhs; + + orLhs.template set<0>(opOr(in01.template get<0>(), spaceMask)); + shuffleRhs.template set<0>(opShuffle(opTable, in01.template get<0>())); + if constexpr (registerCount > 1) { + orLhs.template set<1>(opOr(in01.template get<1>(), spaceMask)); + shuffleRhs.template set<1>(opShuffle(opTable, in01.template get<1>())); + if constexpr (registerCount > 2) { + orLhs.template set<2>(opOr(in01.template get<2>(), spaceMask)); + shuffleRhs.template set<2>(opShuffle(opTable, in01.template get<2>())); + orLhs.template set<3>(opOr(in01.template get<3>(), spaceMask)); + shuffleRhs.template set<3>(opShuffle(opTable, in01.template get<3>())); + } + } + return pod_cmp_eq_op::impl(orLhs, shuffleRhs); + } + }; + + template template JSONIFIER_INLINE void rope_detector::nextScalar( + const pod_simd_array_t in01, const simd_register_t bsRegister, const simd_register_t quoteRegister) noexcept { + const uint64_t escaped = nextEscapeAndTerminalCode(pod_cmp_eq_op::impl(in01, bsRegister)); + const uint64_t quotes = (pod_cmp_eq_op::impl(in01, quoteRegister) & ~escaped); + rope_block::escaped = escaped; + rope_block::quotes = quotes; + return quotes ? finishNext() : finishNextNoInString(); + } + + struct tape_writer_op { + JSONIFIER_INLINE static uint32_t extractIndex(const uint64_t base, const uint64_t bits) noexcept { + return static_cast(postCmpTzcnt(bits) + base); + } + + JSONIFIER_INLINE static uint64_t correctedPopcount(const uint64_t bits) noexcept { + return static_cast(popcount(bits)); + } + + JSONIFIER_INLINE static uint64_t advance(const uint64_t bits) noexcept { + return blsr(bits); + } + }; + + template consteval array generateWhitespaceArray() noexcept { + constexpr const uint8_t values[]{ 0x20u, 0x64u, 0x64u, 0x64u, 0x11u, 0x64u, 0x71u, 0x02u, 0x64u, uint_c_v<'\t'>, uint_c_v<'\n'>, 0x70u, 0x64u, uint_c_v<'\r'>, 0x64u, + 0x64u }; + array returnValues{}; + for (uint64_t x = 0; x < size; ++x) { + returnValues[x] = values[x % 16]; + } + return returnValues; + }; + + template alignas(simdBytesPerRegister) inline constexpr array whitespaceArray{ generateWhitespaceArray() }; + + template consteval array generateOpArray() noexcept { + constexpr const uint8_t values[]{ 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, uint_c_v<':'>, uint_c_v<'{'>, uint_c_v<','>, uint_c_v<'}'>, + 0x00u, 0x00u }; + array returnValues{}; + for (uint64_t x = 0; x < size; ++x) { + returnValues[x] = values[x % 16]; + } + return returnValues; + }; + + template alignas(simdBytesPerRegister) inline constexpr array opArray{ generateOpArray() }; + +#endif + +} diff --git a/deps/jsonifier/include/jsonifier-incl/simd/bit_ops.hpp b/deps/jsonifier/include/jsonifier-incl/simd/bit_ops.hpp new file mode 100644 index 000000000000..03713958752e --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/simd/bit_ops.hpp @@ -0,0 +1,232 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/simd/bit_ops.hpp + */ +#pragma once + +#include + +namespace jsonifier::internal::simd { + + template JSONIFIER_INLINE constexpr uint64_t countlZeroUnsafe(value_type value) noexcept { + if (std::is_constant_evaluated()) { + return static_cast(std::countl_zero(value)); + } else { +#if JSONIFIER_LZCNT && JSONIFIER_ARCH_X64 + return static_cast(__lzcnt16(static_cast(value))); +#elif JSONIFIER_COMPILER_GCC || JSONIFIER_COMPILER_CLANG + return static_cast(__builtin_clz(value) - 16); +#elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_ARM64 + return static_cast(_CountLeadingZeros(value) - 16); +#else + return static_cast(std::countl_zero(value)); +#endif + } + } + + template JSONIFIER_INLINE constexpr uint64_t countlZeroUnsafe(value_type value) noexcept { + if (std::is_constant_evaluated()) { + return static_cast(std::countl_zero(value)); + } else { +#if JSONIFIER_LZCNT && JSONIFIER_ARCH_X64 + return static_cast(_lzcnt_u32(value)); +#elif JSONIFIER_COMPILER_GCC || JSONIFIER_COMPILER_CLANG + return static_cast(__builtin_clz(value)); +#elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_ARM64 + return static_cast(_CountLeadingZeros(value)); +#else + return static_cast(std::countl_zero(value)); +#endif + } + } + + template JSONIFIER_INLINE constexpr uint64_t countlZeroUnsafe(value_type value) noexcept { + if (std::is_constant_evaluated()) { + return static_cast(std::countl_zero(value)); + } else { +#if JSONIFIER_LZCNT && JSONIFIER_ARCH_X64 + return static_cast(_lzcnt_u64(value)); +#elif JSONIFIER_COMPILER_GCC || JSONIFIER_COMPILER_CLANG + return static_cast(__builtin_clzll(value)); +#elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_ARM64 + return static_cast(_CountLeadingZeros64(value)); +#else + return static_cast(std::countl_zero(value)); +#endif + } + } + + template JSONIFIER_INLINE constexpr uint64_t countrZeroUnsafe(value_type value) noexcept { + if (std::is_constant_evaluated()) { + return static_cast(std::countr_zero(value)); + } else { +#if JSONIFIER_BMI && JSONIFIER_ARCH_X64 + return static_cast(_tzcnt_u32(value)); +#elif JSONIFIER_COMPILER_GCC || JSONIFIER_COMPILER_CLANG + return static_cast(__builtin_ctz(value)); +#elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_ARM64 + return static_cast(_CountTrailingZeros(value)); +#else + return static_cast(std::countr_zero(value)); +#endif + } + } + + template JSONIFIER_INLINE constexpr uint64_t countrZeroUnsafe(value_type value) noexcept { + if (std::is_constant_evaluated()) { + return static_cast(std::countr_zero(value)); + } else { +#if JSONIFIER_BMI && JSONIFIER_ARCH_X64 + return static_cast(_tzcnt_u32(value)); +#elif JSONIFIER_COMPILER_GCC || JSONIFIER_COMPILER_CLANG + return static_cast(__builtin_ctz(value)); +#elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_ARM64 + return static_cast(_CountTrailingZeros(value)); +#else + return static_cast(std::countr_zero(value)); +#endif + } + } + + template JSONIFIER_INLINE constexpr uint64_t countrZeroUnsafe(value_type value) noexcept { + if (std::is_constant_evaluated()) { + return static_cast(std::countr_zero(value)); + } else { +#if JSONIFIER_BMI && JSONIFIER_ARCH_X64 + return static_cast(_tzcnt_u64(value)); +#elif JSONIFIER_COMPILER_GCC || JSONIFIER_COMPILER_CLANG + return static_cast(__builtin_ctzll(value)); +#elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_ARM64 + return static_cast(_CountTrailingZeros64(value)); +#else + return static_cast(std::countr_zero(value)); +#endif + } + } + + template JSONIFIER_INLINE constexpr uint64_t countlZero(value_type value) noexcept { + if (std::is_constant_evaluated()) { + return static_cast(std::countl_zero(value)); + } else { +#if JSONIFIER_LZCNT && JSONIFIER_ARCH_X64 + return static_cast(__lzcnt16(static_cast(value))); +#elif JSONIFIER_COMPILER_GCC || JSONIFIER_COMPILER_CLANG + return value == 0 ? 16ULL : static_cast(__builtin_clz(value) - 16); +#elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_ARM64 + return static_cast(_CountLeadingZeros(value) - 16); +#else + return static_cast(std::countl_zero(value)); +#endif + } + } + + template JSONIFIER_INLINE constexpr uint64_t countlZero(value_type value) noexcept { + if (std::is_constant_evaluated()) { + return static_cast(std::countl_zero(value)); + } else { +#if JSONIFIER_LZCNT && JSONIFIER_ARCH_X64 + return static_cast(_lzcnt_u32(value)); +#elif JSONIFIER_COMPILER_GCC || JSONIFIER_COMPILER_CLANG + return value == 0 ? 32ULL : static_cast(__builtin_clz(value)); +#elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_ARM64 + return static_cast(_CountLeadingZeros(value)); +#else + return static_cast(std::countl_zero(value)); +#endif + } + } + + template JSONIFIER_INLINE constexpr uint64_t countlZero(value_type value) noexcept { + if (std::is_constant_evaluated()) { + return static_cast(std::countl_zero(value)); + } else { +#if JSONIFIER_LZCNT && JSONIFIER_ARCH_X64 + return static_cast(_lzcnt_u64(value)); +#elif JSONIFIER_COMPILER_GCC || JSONIFIER_COMPILER_CLANG + return value == 0 ? 64ULL : static_cast(__builtin_clzll(value)); +#elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_ARM64 + return static_cast(_CountLeadingZeros64(value)); +#else + return static_cast(std::countl_zero(value)); +#endif + } + } + + template JSONIFIER_INLINE constexpr uint64_t countrZero(value_type value) noexcept { + if (std::is_constant_evaluated()) { + return static_cast(std::countr_zero(value)); + } else { +#if JSONIFIER_BMI && JSONIFIER_ARCH_X64 + return static_cast(_tzcnt_u32(static_cast(value) | 0x10000U)); +#elif JSONIFIER_COMPILER_GCC || JSONIFIER_COMPILER_CLANG + return static_cast(__builtin_ctz(static_cast(value) | 0x10000U)); +#elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_ARM64 + return static_cast(_CountTrailingZeros(static_cast(value) | 0x10000U)); +#else + return static_cast(std::countr_zero(value)); +#endif + } + } + + template JSONIFIER_INLINE constexpr uint64_t countrZero(value_type value) noexcept { + if (std::is_constant_evaluated()) { + return static_cast(std::countr_zero(value)); + } else { +#if JSONIFIER_BMI && JSONIFIER_ARCH_X64 + return static_cast(_tzcnt_u32(value)); +#elif JSONIFIER_COMPILER_GCC || JSONIFIER_COMPILER_CLANG + return value == 0 ? 32ULL : static_cast(__builtin_ctz(value)); +#elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_ARM64 + return static_cast(_CountTrailingZeros(value)); +#else + return static_cast(std::countr_zero(value)); +#endif + } + } + + template JSONIFIER_INLINE constexpr uint64_t countrZero(value_type value) noexcept { + if (std::is_constant_evaluated()) { + return static_cast(std::countr_zero(value)); + } else { +#if JSONIFIER_BMI && JSONIFIER_ARCH_X64 + return static_cast(_tzcnt_u64(value)); +#elif JSONIFIER_COMPILER_GCC || JSONIFIER_COMPILER_CLANG + return value == 0 ? 64ULL : static_cast(__builtin_ctzll(value)); +#elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_ARM64 + return static_cast(_CountTrailingZeros64(value)); +#else + return static_cast(std::countr_zero(value)); +#endif + } + } + + template JSONIFIER_INLINE constexpr uint64_t popcount(value_type value) noexcept { + if (std::is_constant_evaluated()) { + return static_cast(std::popcount(value)); + } else { +#if JSONIFIER_POPCNT && JSONIFIER_ARCH_X64 + return static_cast(_mm_popcnt_u64(value)); +#elif JSONIFIER_COMPILER_GCC || JSONIFIER_COMPILER_CLANG + return static_cast(__builtin_popcountll(value)); +#elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_ARM64 + return static_cast(_CountOneBits64(value)); +#else + return static_cast(std::popcount(value)); +#endif + } + } + + template JSONIFIER_INLINE constexpr uint64_t blsr(value_type value) noexcept { + if (std::is_constant_evaluated()) { + return static_cast(value & (value - 1)); + } else { +#if JSONIFIER_BMI && JSONIFIER_ARCH_X64 + return static_cast(_blsr_u64(value)); +#else + return static_cast(value & (value - 1)); +#endif + } + } +} diff --git a/deps/jsonifier/include/jsonifier-incl/simd/fallback.hpp b/deps/jsonifier/include/jsonifier-incl/simd/fallback.hpp new file mode 100644 index 000000000000..5f6a753b8572 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/simd/fallback.hpp @@ -0,0 +1,481 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/simd/fallback.hpp + */ +#pragma once + +#include +#include +#include +#include + +namespace jsonifier::internal::simd { + +#if !JSONIFIER_AVX && !JSONIFIER_AVX2 && !JSONIFIER_AVX512 && !JSONIFIER_NEON && !JSONIFIER_SVE2 + + template [[maybe_unused]] JSONIFIER_INLINE uint16_t opCmpEq(simd_int_t01 value, simd_int_t02 other) noexcept { + uint16_t result{}; + result |= static_cast(value.values.xUint8[0] == other.values.xUint8[0]) << 0; + result |= static_cast(value.values.xUint8[1] == other.values.xUint8[1]) << 1; + result |= static_cast(value.values.xUint8[2] == other.values.xUint8[2]) << 2; + result |= static_cast(value.values.xUint8[3] == other.values.xUint8[3]) << 3; + result |= static_cast(value.values.xUint8[4] == other.values.xUint8[4]) << 4; + result |= static_cast(value.values.xUint8[5] == other.values.xUint8[5]) << 5; + result |= static_cast(value.values.xUint8[6] == other.values.xUint8[6]) << 6; + result |= static_cast(value.values.xUint8[7] == other.values.xUint8[7]) << 7; + result |= static_cast(value.values.xUint8[8] == other.values.xUint8[8]) << 8; + result |= static_cast(value.values.xUint8[9] == other.values.xUint8[9]) << 9; + result |= static_cast(value.values.xUint8[10] == other.values.xUint8[10]) << 10; + result |= static_cast(value.values.xUint8[11] == other.values.xUint8[11]) << 11; + result |= static_cast(value.values.xUint8[12] == other.values.xUint8[12]) << 12; + result |= static_cast(value.values.xUint8[13] == other.values.xUint8[13]) << 13; + result |= static_cast(value.values.xUint8[14] == other.values.xUint8[14]) << 14; + result |= static_cast(value.values.xUint8[15] == other.values.xUint8[15]) << 15; + return result; + } + + template [[maybe_unused]] JSONIFIER_INLINE void storeU(simd_int_type_new value, void* storageLocation) noexcept { + pow2MemcpyWrapper<16>(storageLocation, &value.values); + } + + template [[maybe_unused]] JSONIFIER_INLINE void store(simd_int_type_new value, void* storageLocation) noexcept { + pow2MemcpyWrapper<16>(storageLocation, &value.values); + } + + template [[maybe_unused]] JSONIFIER_INLINE simd_int_t01 opSrLi(simd_int_t01 value) noexcept { + simd_int_t01 result; + result.values.xUint16[0] = static_cast(value.values.xUint16[0] >> alignment); + result.values.xUint16[1] = static_cast(value.values.xUint16[1] >> alignment); + result.values.xUint16[2] = static_cast(value.values.xUint16[2] >> alignment); + result.values.xUint16[3] = static_cast(value.values.xUint16[3] >> alignment); + result.values.xUint16[4] = static_cast(value.values.xUint16[4] >> alignment); + result.values.xUint16[5] = static_cast(value.values.xUint16[5] >> alignment); + result.values.xUint16[6] = static_cast(value.values.xUint16[6] >> alignment); + result.values.xUint16[7] = static_cast(value.values.xUint16[7] >> alignment); + return result; + } + + template [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValuesU(const void* str) noexcept { + simd_int_type_new result; + pow2MemcpyWrapper<16>(&result.values, str); + return result; + } + + template [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValues(const void* str) noexcept { + simd_int_type_new result; + pow2MemcpyWrapper<16>(&result.values, str); + return result; + } + + template + [[maybe_unused]] JSONIFIER_INLINE simd_int_t01 opAlignR(simd_int_t01 value, simd_int_t02 other) noexcept { + simd_int_t01 result; + { + static constexpr int64_t src = static_cast(0) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[0] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[0] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[0] = 0; + } + } + { + constexpr int64_t src = static_cast(1) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[1] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[1] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[1] = 0; + } + } + { + constexpr int64_t src = static_cast(2) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[2] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[2] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[2] = 0; + } + } + { + constexpr int64_t src = static_cast(3) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[3] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[3] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[3] = 0; + } + } + { + constexpr int64_t src = static_cast(4) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[4] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[4] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[4] = 0; + } + } + { + constexpr int64_t src = static_cast(5) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[5] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[5] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[5] = 0; + } + } + { + constexpr int64_t src = static_cast(6) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[6] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[6] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[6] = 0; + } + } + { + constexpr int64_t src = static_cast(7) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[7] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[7] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[7] = 0; + } + } + { + constexpr int64_t src = static_cast(8) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[8] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[8] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[8] = 0; + } + } + { + constexpr int64_t src = static_cast(9) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[9] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[9] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[9] = 0; + } + } + { + constexpr int64_t src = static_cast(10) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[10] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[10] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[10] = 0; + } + } + { + constexpr int64_t src = static_cast(11) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[11] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[11] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[11] = 0; + } + } + { + constexpr int64_t src = static_cast(12) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[12] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[12] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[12] = 0; + } + } + { + constexpr int64_t src = static_cast(13) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[13] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[13] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[13] = 0; + } + } + { + constexpr int64_t src = static_cast(14) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[14] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[14] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[14] = 0; + } + } + { + constexpr int64_t src = static_cast(15) + static_cast(alignment); + if constexpr (src >= 0 && src < static_cast(simd_int_t01::eightPer)) { + result.values.xUint8[15] = other.values.xUint8[static_cast(src)]; + } else if constexpr (src >= static_cast(simd_int_t01::eightPer) && src < static_cast(simd_int_t01::eightPer) * 2) { + result.values.xUint8[15] = value.values.xUint8[static_cast(src - static_cast(simd_int_t01::eightPer))]; + } else { + result.values.xUint8[15] = 0; + } + } + return result; + } + + template [[maybe_unused]] JSONIFIER_INLINE uint64_t postCmpTzcntUnsafe(const value_type value) noexcept { + return countrZeroUnsafe(value); + } + + template [[maybe_unused]] JSONIFIER_INLINE uint16_t opBitMask(simd_int_t01 value) noexcept { + uint64_t rawLo = value.values.xUint64[0]; + uint64_t rawHi = value.values.xUint64[1]; + if constexpr (std::endian::native == std::endian::big) { + rawLo = internal::byteswap(rawLo); + rawHi = internal::byteswap(rawHi); + } + uint64_t highBits0 = rawLo & 0x8080808080808080ull; + uint64_t highBits1 = rawHi & 0x8080808080808080ull; + uint16_t mask0 = static_cast((highBits0 * 0x0002040810204081ull) >> 56); + uint16_t mask1 = static_cast((highBits1 * 0x0002040810204081ull) >> 56); + return static_cast(mask0 | (mask1 << 8)); + } + + template [[maybe_unused]] JSONIFIER_INLINE uint16_t opBitMaskRaw(simd_int_t01 value) noexcept { + return opBitMask(value); + } + + template [[maybe_unused]] JSONIFIER_INLINE uint64_t postCmpTzcnt(const value_type value) noexcept { + return countrZero(value); + } + + template [[maybe_unused]] JSONIFIER_INLINE bool opTest(simd_int_t01 value) noexcept { + if (value.values.xUint64[0] != 0) { + return false; + } + if (value.values.xUint64[1] != 0) { + return false; + } + return true; + } + + template + [[maybe_unused]] JSONIFIER_INLINE simd_int_t01 opPrev(simd_int_t01 current, simd_int_t02 previous) noexcept { + return opAlignR(current, previous); + } + + template + [[maybe_unused]] JSONIFIER_INLINE simd_int_t01 opCmpEqRaw(simd_int_t01 value, simd_int_t02 other) noexcept { + simd_int_t01 result; + result.values.xUint8[0] = (value.values.xUint8[0] == other.values.xUint8[0]) ? 0xFFu : 0x00u; + result.values.xUint8[1] = (value.values.xUint8[1] == other.values.xUint8[1]) ? 0xFFu : 0x00u; + result.values.xUint8[2] = (value.values.xUint8[2] == other.values.xUint8[2]) ? 0xFFu : 0x00u; + result.values.xUint8[3] = (value.values.xUint8[3] == other.values.xUint8[3]) ? 0xFFu : 0x00u; + result.values.xUint8[4] = (value.values.xUint8[4] == other.values.xUint8[4]) ? 0xFFu : 0x00u; + result.values.xUint8[5] = (value.values.xUint8[5] == other.values.xUint8[5]) ? 0xFFu : 0x00u; + result.values.xUint8[6] = (value.values.xUint8[6] == other.values.xUint8[6]) ? 0xFFu : 0x00u; + result.values.xUint8[7] = (value.values.xUint8[7] == other.values.xUint8[7]) ? 0xFFu : 0x00u; + result.values.xUint8[8] = (value.values.xUint8[8] == other.values.xUint8[8]) ? 0xFFu : 0x00u; + result.values.xUint8[9] = (value.values.xUint8[9] == other.values.xUint8[9]) ? 0xFFu : 0x00u; + result.values.xUint8[10] = (value.values.xUint8[10] == other.values.xUint8[10]) ? 0xFFu : 0x00u; + result.values.xUint8[11] = (value.values.xUint8[11] == other.values.xUint8[11]) ? 0xFFu : 0x00u; + result.values.xUint8[12] = (value.values.xUint8[12] == other.values.xUint8[12]) ? 0xFFu : 0x00u; + result.values.xUint8[13] = (value.values.xUint8[13] == other.values.xUint8[13]) ? 0xFFu : 0x00u; + result.values.xUint8[14] = (value.values.xUint8[14] == other.values.xUint8[14]) ? 0xFFu : 0x00u; + result.values.xUint8[15] = (value.values.xUint8[15] == other.values.xUint8[15]) ? 0xFFu : 0x00u; + return result; + } + + template + [[maybe_unused]] JSONIFIER_INLINE simd_int_t01 opCmpLtRaw(simd_int_t01 value, simd_int_t02 other) noexcept { + simd_int_t01 result; + result.values.xUint8[0] = (value.values.xUint8[0] < other.values.xUint8[0]) ? 0xFFu : 0x00u; + result.values.xUint8[1] = (value.values.xUint8[1] < other.values.xUint8[1]) ? 0xFFu : 0x00u; + result.values.xUint8[2] = (value.values.xUint8[2] < other.values.xUint8[2]) ? 0xFFu : 0x00u; + result.values.xUint8[3] = (value.values.xUint8[3] < other.values.xUint8[3]) ? 0xFFu : 0x00u; + result.values.xUint8[4] = (value.values.xUint8[4] < other.values.xUint8[4]) ? 0xFFu : 0x00u; + result.values.xUint8[5] = (value.values.xUint8[5] < other.values.xUint8[5]) ? 0xFFu : 0x00u; + result.values.xUint8[6] = (value.values.xUint8[6] < other.values.xUint8[6]) ? 0xFFu : 0x00u; + result.values.xUint8[7] = (value.values.xUint8[7] < other.values.xUint8[7]) ? 0xFFu : 0x00u; + result.values.xUint8[8] = (value.values.xUint8[8] < other.values.xUint8[8]) ? 0xFFu : 0x00u; + result.values.xUint8[9] = (value.values.xUint8[9] < other.values.xUint8[9]) ? 0xFFu : 0x00u; + result.values.xUint8[10] = (value.values.xUint8[10] < other.values.xUint8[10]) ? 0xFFu : 0x00u; + result.values.xUint8[11] = (value.values.xUint8[11] < other.values.xUint8[11]) ? 0xFFu : 0x00u; + result.values.xUint8[12] = (value.values.xUint8[12] < other.values.xUint8[12]) ? 0xFFu : 0x00u; + result.values.xUint8[13] = (value.values.xUint8[13] < other.values.xUint8[13]) ? 0xFFu : 0x00u; + result.values.xUint8[14] = (value.values.xUint8[14] < other.values.xUint8[14]) ? 0xFFu : 0x00u; + result.values.xUint8[15] = (value.values.xUint8[15] < other.values.xUint8[15]) ? 0xFFu : 0x00u; + return result; + } + + template + [[maybe_unused]] JSONIFIER_INLINE simd_int_t01 opShuffle(simd_int_t01 value, simd_int_t02 other) noexcept { + simd_int_t01 result; + uint8_t idx = other.values.xUint8[0]; + result.values.xUint8[0] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + idx = other.values.xUint8[1]; + result.values.xUint8[1] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + idx = other.values.xUint8[2]; + result.values.xUint8[2] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + idx = other.values.xUint8[3]; + result.values.xUint8[3] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + idx = other.values.xUint8[4]; + result.values.xUint8[4] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + idx = other.values.xUint8[5]; + result.values.xUint8[5] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + idx = other.values.xUint8[6]; + result.values.xUint8[6] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + idx = other.values.xUint8[7]; + result.values.xUint8[7] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + idx = other.values.xUint8[8]; + result.values.xUint8[8] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + idx = other.values.xUint8[9]; + result.values.xUint8[9] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + idx = other.values.xUint8[10]; + result.values.xUint8[10] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + idx = other.values.xUint8[11]; + result.values.xUint8[11] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + idx = other.values.xUint8[12]; + result.values.xUint8[12] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + idx = other.values.xUint8[13]; + result.values.xUint8[13] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + idx = other.values.xUint8[14]; + result.values.xUint8[14] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + idx = other.values.xUint8[15]; + result.values.xUint8[15] = (idx & 0x80u) ? 0x00u : value.values.xUint8[idx & 0x0Fu]; + return result; + } + + template + [[maybe_unused]] JSONIFIER_INLINE simd_int_t01 opSubs(simd_int_t01 value, simd_int_t02 other) noexcept { + simd_int_t01 result; + uint16_t a = value.values.xUint8[0]; + uint16_t b = other.values.xUint8[0]; + result.values.xUint8[0] = static_cast(a > b ? a - b : 0); + a = value.values.xUint8[1]; + b = other.values.xUint8[1]; + result.values.xUint8[1] = static_cast(a > b ? a - b : 0); + a = value.values.xUint8[2]; + b = other.values.xUint8[2]; + result.values.xUint8[2] = static_cast(a > b ? a - b : 0); + a = value.values.xUint8[3]; + b = other.values.xUint8[3]; + result.values.xUint8[3] = static_cast(a > b ? a - b : 0); + a = value.values.xUint8[4]; + b = other.values.xUint8[4]; + result.values.xUint8[4] = static_cast(a > b ? a - b : 0); + a = value.values.xUint8[5]; + b = other.values.xUint8[5]; + result.values.xUint8[5] = static_cast(a > b ? a - b : 0); + a = value.values.xUint8[6]; + b = other.values.xUint8[6]; + result.values.xUint8[6] = static_cast(a > b ? a - b : 0); + a = value.values.xUint8[7]; + b = other.values.xUint8[7]; + result.values.xUint8[7] = static_cast(a > b ? a - b : 0); + a = value.values.xUint8[8]; + b = other.values.xUint8[8]; + result.values.xUint8[8] = static_cast(a > b ? a - b : 0); + a = value.values.xUint8[9]; + b = other.values.xUint8[9]; + result.values.xUint8[9] = static_cast(a > b ? a - b : 0); + a = value.values.xUint8[10]; + b = other.values.xUint8[10]; + result.values.xUint8[10] = static_cast(a > b ? a - b : 0); + a = value.values.xUint8[11]; + b = other.values.xUint8[11]; + result.values.xUint8[11] = static_cast(a > b ? a - b : 0); + a = value.values.xUint8[12]; + b = other.values.xUint8[12]; + result.values.xUint8[12] = static_cast(a > b ? a - b : 0); + a = value.values.xUint8[13]; + b = other.values.xUint8[13]; + result.values.xUint8[13] = static_cast(a > b ? a - b : 0); + a = value.values.xUint8[14]; + b = other.values.xUint8[14]; + result.values.xUint8[14] = static_cast(a > b ? a - b : 0); + a = value.values.xUint8[15]; + b = other.values.xUint8[15]; + result.values.xUint8[15] = static_cast(a > b ? a - b : 0); + return result; + } + + template + [[maybe_unused]] JSONIFIER_INLINE simd_int_t01 opXor(simd_int_t01 value, simd_int_t02 other) noexcept { + simd_int_t01 result; + result.values.xUint64[0] = value.values.xUint64[0] ^ other.values.xUint64[0]; + result.values.xUint64[1] = value.values.xUint64[1] ^ other.values.xUint64[1]; + return result; + } + + template + [[maybe_unused]] JSONIFIER_INLINE simd_int_t01 opAnd(simd_int_t01 value, simd_int_t02 other) noexcept { + simd_int_t01 result; + result.values.xUint64[0] = value.values.xUint64[0] & other.values.xUint64[0]; + result.values.xUint64[1] = value.values.xUint64[1] & other.values.xUint64[1]; + return result; + } + + template + [[maybe_unused]] JSONIFIER_INLINE simd_int_t01 opOr(simd_int_t01 value, simd_int_t02 other) noexcept { + simd_int_t01 result; + result.values.xUint64[0] = value.values.xUint64[0] | other.values.xUint64[0]; + result.values.xUint64[1] = value.values.xUint64[1] | other.values.xUint64[1]; + return result; + } + + template JSONIFIER_INLINE simd_int_t01 orAll(simd_array chunks) noexcept { + return opOr(chunks.template get<0>(), opOr(chunks.template get<1>(), opOr(chunks.template get<2>(), chunks.template get<3>()))); + } + + template + requires(sizeof(char_t) == 8) + [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValue(const char_t str) noexcept { + simd_int_type_new result; + result.values.xUint64[0] = static_cast(str); + result.values.xUint64[1] = static_cast(str); + return result; + } + + template + requires(sizeof(char_t) == 1) + [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValue(const char_t str) noexcept { + simd_int_type_new result; + result.values.xUint8[0] = static_cast(str); + result.values.xUint8[1] = static_cast(str); + result.values.xUint8[2] = static_cast(str); + result.values.xUint8[3] = static_cast(str); + result.values.xUint8[4] = static_cast(str); + result.values.xUint8[5] = static_cast(str); + result.values.xUint8[6] = static_cast(str); + result.values.xUint8[7] = static_cast(str); + result.values.xUint8[8] = static_cast(str); + result.values.xUint8[9] = static_cast(str); + result.values.xUint8[10] = static_cast(str); + result.values.xUint8[11] = static_cast(str); + result.values.xUint8[12] = static_cast(str); + result.values.xUint8[13] = static_cast(str); + result.values.xUint8[14] = static_cast(str); + result.values.xUint8[15] = static_cast(str); + return result; + } + + template JSONIFIER_INLINE bool isAscii(simd_type value) noexcept { + return opBitMask(value) == 0; + } + +#endif + +} diff --git a/deps/jsonifier/include/jsonifier-incl/simd/jsonifier_cpu_instructions.hpp b/deps/jsonifier/include/jsonifier-incl/simd/jsonifier_cpu_instructions.hpp new file mode 100644 index 000000000000..d72e71e533e1 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/simd/jsonifier_cpu_instructions.hpp @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/simd/jsonifier_cpu_instructions.hpp + */ +#pragma once + +#undef JSONIFIER_SVE2_VECTOR_BITS +#define JSONIFIER_SVE2_VECTOR_BITS 0 + +#if !defined(JSONIFIER_LZCNT) + #define JSONIFIER_LZCNT 1 +#endif + +#if !defined(JSONIFIER_POPCNT) + #define JSONIFIER_POPCNT 1 +#endif + +#if !defined(JSONIFIER_BMI) + #define JSONIFIER_BMI 1 +#endif + +#if !defined(JSONIFIER_CLMUL) + #define JSONIFIER_CLMUL 1 +#endif + +#if !defined(JSONIFIER_NEON) + #define JSONIFIER_NEON 0 +#endif + +#if !defined(JSONIFIER_AVX) + #define JSONIFIER_AVX 1 +#endif + +#if !defined(JSONIFIER_AVX2) + #define JSONIFIER_AVX2 1 +#endif + +#if !defined(JSONIFIER_AVX512) + #define JSONIFIER_AVX512 0 +#endif + +#if !defined(JSONIFIER_SVE2) + #define JSONIFIER_SVE2 0 +#endif + +#if !defined(JSONIFIER_ANY_AVX) + #define JSONIFIER_ANY_AVX (JSONIFIER_AVX || JSONIFIER_AVX2 || JSONIFIER_AVX512) +#endif + +#if !defined(JSONIFIER_ANY_SIMD) + #define JSONIFIER_ANY_SIMD (JSONIFIER_AVX || JSONIFIER_AVX2 || JSONIFIER_AVX512 || JSONIFIER_NEON || JSONIFIER_SVE2) +#endif + +#if JSONIFIER_NEON && JSONIFIER_SVE2 + #error "JSONIFIER_NEON and JSONIFIER_SVE2 are mutually exclusive backends - every SVE2 part also reports NEON, so exactly one must be selected." +#endif + +#if JSONIFIER_SVE2 && JSONIFIER_SVE2_VECTOR_BITS == 0 + #error "JSONIFIER_SVE2 is selected but JSONIFIER_SVE2_VECTOR_BITS is 0 - the fixed-length SVE2 typedefs require a measured vector length." +#endif diff --git a/deps/jsonifier/include/jsonifier-incl/simd/jsonifier_cpu_properties.hpp b/deps/jsonifier/include/jsonifier-incl/simd/jsonifier_cpu_properties.hpp new file mode 100644 index 000000000000..261cc1de53b0 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/simd/jsonifier_cpu_properties.hpp @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/simd/jsonifier_cpu_properties.hpp.in + */ +#pragma once + +#include +#include + +#if !defined(JSONIFIER_CPU_ALIGNMENT) + #define JSONIFIER_CPU_ALIGNMENT 32 +#endif + +namespace jsonifier::internal { + + enum class cpu_property_types : uint64_t { + alignment, + arg_alignment, + }; + + struct cpu_properties { + public: + static constexpr uint64_t values[]{ JSONIFIER_CPU_ALIGNMENT, 64ULL }; + + static consteval uint64_t getValue(cpu_property_types index) { + return values[static_cast(index)]; + } + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/simd/neon.hpp b/deps/jsonifier/include/jsonifier-incl/simd/neon.hpp new file mode 100644 index 000000000000..b0a10835a3a1 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/simd/neon.hpp @@ -0,0 +1,125 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/simd/neon.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::internal::simd { + +#if JSONIFIER_NEON + + template [[maybe_unused]] JSONIFIER_INLINE auto opCmpEqRaw(simd_int_t01 value, simd_int_t02 other) noexcept { + return vceqq_u8(value, other); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opCmpLtRaw(simd_int_t01 value, simd_int_t02 other) noexcept { + return vcgtq_u8(other, value); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opShuffle(simd_int_t01 value, simd_int_t02 other) noexcept { + const auto bitMask{ vdupq_n_u8(0x0F) }; + return vqtbl1q_u8(value, vandq_u8(other, bitMask)); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opCmpEq(simd_int_t01 value, simd_int_t02 other) noexcept { + return vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(vceqq_u8(value, other)), 4)), 0); + } + + template [[maybe_unused]] JSONIFIER_INLINE void storeU(simd_int_type_new value, void* __restrict storageLocation) noexcept { + vst1q_u8(static_cast(storageLocation), value); + } + + template [[maybe_unused]] JSONIFIER_INLINE void store(simd_int_type_new value, void* __restrict storageLocation) noexcept { + vst1q_u8(static_cast(storageLocation), value); + } + + template [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValuesU(const void* __restrict str) noexcept { + return vld1q_u8(static_cast(str)); + } + + template [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValues(const void* __restrict str) noexcept { + return vld1q_u8(static_cast(str)); + } + + template [[maybe_unused]] JSONIFIER_INLINE auto opSrLi(simd_int_t01 value) noexcept { + return vshrq_n_u8(value, alignment); + } + + template [[maybe_unused]] JSONIFIER_INLINE uint16_t opBitMask(simd_int_t01 value) noexcept { + static constexpr uint8x16_t bitMask{ 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80 }; + const auto minput = vandq_u8(value, bitMask); + uint8x16_t tmp = vpaddq_u8(minput, minput); + tmp = vpaddq_u8(tmp, tmp); + tmp = vpaddq_u8(tmp, tmp); + return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); + } + + template [[maybe_unused]] JSONIFIER_INLINE uint64_t opBitMaskRaw(simd_int_t01 value) noexcept { + return vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(value), 4)), 0); + } + + template [[maybe_unused]] JSONIFIER_INLINE uint64_t postCmpTzcntUnsafe(value_type value) noexcept { + return countrZeroUnsafe(value) >> 2; + } + + template [[maybe_unused]] JSONIFIER_INLINE bool opTest(simd_int_t01 value) noexcept { + return vget_lane_u64(vreinterpret_u64_u8(vqmovn_u16(vreinterpretq_u16_u8(value))), 0) == 0; + } + + template [[maybe_unused]] JSONIFIER_INLINE uint64_t postCmpTzcnt(value_type value) noexcept { + return countrZero(value) >> 2; + } + + template + [[maybe_unused]] JSONIFIER_INLINE jsonifier_simd_int_128 opSubs(simd_int_t01 value, simd_int_t02 other) noexcept { + return vqsubq_u8(value, other); + } + + template + [[maybe_unused]] JSONIFIER_INLINE jsonifier_simd_int_128 opXor(simd_int_t01 value, simd_int_t02 other) noexcept { + return veorq_u8(value, other); + } + + template + [[maybe_unused]] JSONIFIER_INLINE jsonifier_simd_int_128 opAnd(simd_int_t01 value, simd_int_t02 other) noexcept { + return vandq_u8(value, other); + } + + template + [[maybe_unused]] JSONIFIER_INLINE jsonifier_simd_int_128 opOr(simd_int_t01 value, simd_int_t02 other) noexcept { + return vorrq_u8(value, other); + } + + template JSONIFIER_INLINE simd_int_t01 orAll(const simd_array chunks) noexcept { + return opOr(chunks.template get<0>(), opOr(chunks.template get<1>(), opOr(chunks.template get<2>(), chunks.template get<3>()))); + } + + template + [[maybe_unused]] JSONIFIER_INLINE auto opPrev(simd_int_t01 current, simd_int_t02 previous) noexcept { + return vextq_u8(previous, current, shift); + } + + template JSONIFIER_INLINE bool isAscii(simd_int_t input) noexcept { + return opTest(vandq_u8(input, vdupq_n_u8(0x80))); + } + + template + requires(sizeof(char_t) == 8) + [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValue(char_t str) noexcept { + return vdupq_n_u64(str); + } + + template + requires(sizeof(char_t) == 1) + [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValue(char_t str) noexcept { + return vdupq_n_u8(static_cast(str)); + } + +#endif + +} diff --git a/deps/jsonifier/include/jsonifier-incl/simd/neon_stage1.hpp b/deps/jsonifier/include/jsonifier-incl/simd/neon_stage1.hpp new file mode 100644 index 000000000000..f520d83275fa --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/simd/neon_stage1.hpp @@ -0,0 +1,335 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/simd/neon_stage1.hpp + */ +// Sampled from Dr. Lemire's library, simdjson: https://github.com/simdjson/simdjson +#pragma once + +#include + +namespace jsonifier::internal::simd { + +#if JSONIFIER_NEON + + inline constexpr array shiftAmounts{ [] { + array returnValue{}; + for (uint64_t x = 0; x < simdRegistersPerBlock; ++x) { + returnValue[x] = simdBytesPerRegister * x; + } + return returnValue; + }() }; + + consteval uint64_t getShiftAmount(const uint64_t index) noexcept { + return shiftAmounts[index % shiftAmounts.size()]; + } + + struct prefix_xor_op { + JSONIFIER_INLINE static uint64_t impl(uint64_t bitmask) noexcept { + #if JSONIFIER_CLMUL && JSONIFIER_COMPILER_CLANG + return vgetq_lane_u64(vreinterpretq_u64_p128(vmull_p64(~0ULL, bitmask)), 0); + #else + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; + #endif + } + }; + + struct unescaped_collector { + JSONIFIER_INLINE static uint64_t impl(const simd_array_t in01) noexcept { + static constexpr uint8x16_t bitMask{ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 }; + uint8x16_t sum0 = vpaddq_u8(vandq_u8(in01.template get<0>(), bitMask), vandq_u8(in01.template get<1>(), bitMask)); + uint8x16_t sum1 = vpaddq_u8(vandq_u8(in01.template get<2>(), bitMask), vandq_u8(in01.template get<3>(), bitMask)); + sum0 = vpaddq_u8(sum0, sum1); + sum0 = vpaddq_u8(sum0, sum0); + return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0); + } + }; + + struct ws_collector { + JSONIFIER_INLINE static uint64_t impl(const simd_array_t in01, const jsonifier_simd_int_t whitespaceTableLocal) noexcept { + const uint8x16_t d00 = in01.template get<0>(); + const uint8x16_t d01 = in01.template get<1>(); + const uint8x16_t d02 = in01.template get<2>(); + const uint8x16_t d03 = in01.template get<3>(); + const uint8x16_t matchWs0 = vqtbx1q_u8(vceqq_u8(d00, vdupq_n_u8(uint_c_v<' '>)), whitespaceTableLocal, d00); + const uint8x16_t matchWs1 = vqtbx1q_u8(vceqq_u8(d01, vdupq_n_u8(uint_c_v<' '>)), whitespaceTableLocal, d01); + const uint8x16_t matchWs2 = vqtbx1q_u8(vceqq_u8(d02, vdupq_n_u8(uint_c_v<' '>)), whitespaceTableLocal, d02); + const uint8x16_t matchWs3 = vqtbx1q_u8(vceqq_u8(d03, vdupq_n_u8(uint_c_v<' '>)), whitespaceTableLocal, d03); + static constexpr uint8x16_t bitMask{ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 }; + uint8x16_t wsSum0 = vpaddq_u8(vandq_u8(matchWs0, bitMask), vandq_u8(matchWs1, bitMask)); + uint8x16_t wsSum1 = vpaddq_u8(vandq_u8(matchWs2, bitMask), vandq_u8(matchWs3, bitMask)); + wsSum0 = vpaddq_u8(wsSum0, wsSum1); + wsSum0 = vpaddq_u8(wsSum0, wsSum0); + return vgetq_lane_u64(vreinterpretq_u64_u8(wsSum0), 0); + } + }; + + struct op_collector { + JSONIFIER_INLINE static uint64_t impl(const simd_array_t in01, const jsonifier_simd_int_t opTable, const jsonifier_simd_int_t) noexcept { + const uint8x16_t d00 = in01.template get<0>(); + const uint8x16_t d01 = in01.template get<1>(); + const uint8x16_t d02 = in01.template get<2>(); + const uint8x16_t d03 = in01.template get<3>(); + const uint8x16_t matchOp0 = vceqq_u8(vqtbl1q_u8(opTable, vshrq_n_u8(vaddq_u8(d00, vdupq_n_u8(3)), 4)), d00); + const uint8x16_t matchOp1 = vceqq_u8(vqtbl1q_u8(opTable, vshrq_n_u8(vaddq_u8(d01, vdupq_n_u8(3)), 4)), d01); + const uint8x16_t matchOp2 = vceqq_u8(vqtbl1q_u8(opTable, vshrq_n_u8(vaddq_u8(d02, vdupq_n_u8(3)), 4)), d02); + const uint8x16_t matchOp3 = vceqq_u8(vqtbl1q_u8(opTable, vshrq_n_u8(vaddq_u8(d03, vdupq_n_u8(3)), 4)), d03); + static constexpr uint8x16_t bitMask{ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 }; + uint8x16_t opSum0 = vpaddq_u8(vandq_u8(matchOp0, bitMask), vandq_u8(matchOp1, bitMask)); + uint8x16_t opSum1 = vpaddq_u8(vandq_u8(matchOp2, bitMask), vandq_u8(matchOp3, bitMask)); + opSum0 = vpaddq_u8(opSum0, opSum1); + opSum0 = vpaddq_u8(opSum0, opSum0); + return vgetq_lane_u64(vreinterpretq_u64_u8(opSum0), 0); + } + }; + + struct bracket_masks { + uint64_t closes; + uint64_t opens; + }; + + struct bracket_collector { + JSONIFIER_INLINE static uint8x16_t partialBitmask(const uint8x16_t f0, const uint8x16_t f1, const uint8x16_t f2, const uint8x16_t f3, const uint8x16_t target) noexcept { + static constexpr uint8x16_t bitMask{ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 }; + const uint8x16_t sum0 = vpaddq_u8(vandq_u8(vceqq_u8(f0, target), bitMask), vandq_u8(vceqq_u8(f1, target), bitMask)); + const uint8x16_t sum1 = vpaddq_u8(vandq_u8(vceqq_u8(f2, target), bitMask), vandq_u8(vceqq_u8(f3, target), bitMask)); + return vpaddq_u8(sum0, sum1); + } + + JSONIFIER_INLINE static bracket_masks impl(const simd_array_t in01, const uint8x16_t spaceMask, const uint8x16_t openRegister, const uint8x16_t closeRegister) noexcept { + const uint8x16_t f0 = vorrq_u8(in01.template get<0>(), spaceMask); + const uint8x16_t f1 = vorrq_u8(in01.template get<1>(), spaceMask); + const uint8x16_t f2 = vorrq_u8(in01.template get<2>(), spaceMask); + const uint8x16_t f3 = vorrq_u8(in01.template get<3>(), spaceMask); + const uint64x2_t pairedMasks = vreinterpretq_u64_u8(vpaddq_u8(partialBitmask(f0, f1, f2, f3, openRegister), partialBitmask(f0, f1, f2, f3, closeRegister))); + return { vgetq_lane_u64(pairedMasks, 1), vgetq_lane_u64(pairedMasks, 0) }; + } + }; + + struct ws_op_collector { + JSONIFIER_INLINE static uint64x2_t impl(const simd_array_t in01, const jsonifier_simd_int_t opTable, const jsonifier_simd_int_t, + const jsonifier_simd_int_t whitespaceTableLocal) noexcept { + static constexpr uint8x16_t bitMask{ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 }; + const uint8x16_t d00 = in01.template get<0>(); + const uint8x16_t d01 = in01.template get<1>(); + const uint8x16_t d02 = in01.template get<2>(); + const uint8x16_t d03 = in01.template get<3>(); + const uint8x16_t matchWs0 = vqtbx1q_u8(vceqq_u8(d00, vdupq_n_u8(uint_c_v<' '>)), whitespaceTableLocal, d00); + const uint8x16_t matchWs1 = vqtbx1q_u8(vceqq_u8(d01, vdupq_n_u8(uint_c_v<' '>)), whitespaceTableLocal, d01); + const uint8x16_t matchWs2 = vqtbx1q_u8(vceqq_u8(d02, vdupq_n_u8(uint_c_v<' '>)), whitespaceTableLocal, d02); + const uint8x16_t matchWs3 = vqtbx1q_u8(vceqq_u8(d03, vdupq_n_u8(uint_c_v<' '>)), whitespaceTableLocal, d03); + const uint8x16_t matchOp0 = vceqq_u8(vqtbl1q_u8(opTable, vshrq_n_u8(vaddq_u8(d00, vdupq_n_u8(3)), 4)), d00); + const uint8x16_t matchOp1 = vceqq_u8(vqtbl1q_u8(opTable, vshrq_n_u8(vaddq_u8(d01, vdupq_n_u8(3)), 4)), d01); + const uint8x16_t matchOp2 = vceqq_u8(vqtbl1q_u8(opTable, vshrq_n_u8(vaddq_u8(d02, vdupq_n_u8(3)), 4)), d02); + const uint8x16_t matchOp3 = vceqq_u8(vqtbl1q_u8(opTable, vshrq_n_u8(vaddq_u8(d03, vdupq_n_u8(3)), 4)), d03); + const uint8x16_t wsSum = + vpaddq_u8(vpaddq_u8(vandq_u8(matchWs0, bitMask), vandq_u8(matchWs1, bitMask)), vpaddq_u8(vandq_u8(matchWs2, bitMask), vandq_u8(matchWs3, bitMask))); + const uint8x16_t opSum = + vpaddq_u8(vpaddq_u8(vandq_u8(matchOp0, bitMask), vandq_u8(matchOp1, bitMask)), vpaddq_u8(vandq_u8(matchOp2, bitMask), vandq_u8(matchOp3, bitMask))); + return vreinterpretq_u64_u8(vpaddq_u8(wsSum, opSum)); + } + }; + + template struct scalar_bitmask_collector { + using simd_array_type = pod_simd_array_t; + + JSONIFIER_INLINE static uint64_t impl(const simd_array_type matches) noexcept { + static constexpr uint8x16_t bitMask{ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 }; + if constexpr (registerCount == 1) { + const uint8x16_t masked = vandq_u8(matches.template get<0>(), bitMask); + uint8x16_t sum = vpaddq_u8(masked, masked); + sum = vpaddq_u8(sum, sum); + sum = vpaddq_u8(sum, sum); + return static_cast(vgetq_lane_u16(vreinterpretq_u16_u8(sum), 0)); + } else { + uint8x16_t sum = vpaddq_u8(vandq_u8(matches.template get<0>(), bitMask), vandq_u8(matches.template get<1>(), bitMask)); + sum = vpaddq_u8(sum, sum); + sum = vpaddq_u8(sum, sum); + return static_cast(vgetq_lane_u32(vreinterpretq_u32_u8(sum), 0)); + } + } + + static_assert(registerCount == 1 || registerCount == 2); + }; + + template struct pod_cmp_eq_op { + using simd_array_type = pod_simd_array_t; + + JSONIFIER_INLINE static uint64_t impl(const simd_array_type in01, const uint8x16_t rhsBroadcast) noexcept { + simd_array_type matches; + matches.template set<0>(vceqq_u8(in01.template get<0>(), rhsBroadcast)); + if constexpr (registerCount > 1) { + matches.template set<1>(vceqq_u8(in01.template get<1>(), rhsBroadcast)); + } + return scalar_bitmask_collector::impl(matches); + } + + static_assert(registerBytes == 16 && (registerCount == 1 || registerCount == 2)); + }; + + template struct pod_ws_collector { + using simd_array_type = pod_simd_array_t; + + JSONIFIER_INLINE static uint64_t impl(const simd_array_type in01, const uint8x16_t whitespaceTableLocal) noexcept { + if constexpr (registerCount == simdRegistersPerBlock) { + return ws_collector::impl(in01, whitespaceTableLocal); + } else { + simd_array_type matches; + const uint8x16_t d00 = in01.template get<0>(); + matches.template set<0>(vqtbx1q_u8(vceqq_u8(d00, vdupq_n_u8(uint_c_v<' '>)), whitespaceTableLocal, d00)); + if constexpr (registerCount > 1) { + const uint8x16_t d01 = in01.template get<1>(); + matches.template set<1>(vqtbx1q_u8(vceqq_u8(d01, vdupq_n_u8(uint_c_v<' '>)), whitespaceTableLocal, d01)); + } + return scalar_bitmask_collector::impl(matches); + } + } + }; + + template struct scalar_op_collector { + using simd_array_type = pod_simd_array_t; + + JSONIFIER_INLINE static uint64_t impl(const simd_array_type in01, const uint8x16_t opTable, const uint8x16_t spaceMask) noexcept { + if constexpr (registerCount == simdRegistersPerBlock) { + return op_collector::impl(in01, opTable, spaceMask); + } else { + simd_array_type matches; + const uint8x16_t d00 = in01.template get<0>(); + matches.template set<0>(vceqq_u8(vqtbl1q_u8(opTable, vshrq_n_u8(vaddq_u8(d00, vdupq_n_u8(3)), 4)), d00)); + if constexpr (registerCount > 1) { + const uint8x16_t d01 = in01.template get<1>(); + matches.template set<1>(vceqq_u8(vqtbl1q_u8(opTable, vshrq_n_u8(vaddq_u8(d01, vdupq_n_u8(3)), 4)), d01)); + } + return scalar_bitmask_collector::impl(matches); + } + } + }; + + template struct rope_detector : rope_block { + JSONIFIER_INLINE void next(const simd_array_t in01, const jsonifier_simd_int_t bsRegister, const jsonifier_simd_int_t quoteRegister) noexcept { + static constexpr uint8x16_t bitMask{ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 }; + const uint8x16_t d0 = in01.template get<0>(); + const uint8x16_t d1 = in01.template get<1>(); + const uint8x16_t d2 = in01.template get<2>(); + const uint8x16_t d3 = in01.template get<3>(); + const uint8x16_t backslashPartial = toPartialBitmask(vceqq_u8(d0, bsRegister), vceqq_u8(d1, bsRegister), vceqq_u8(d2, bsRegister), vceqq_u8(d3, bsRegister), bitMask); + const uint8x16_t quotesPartial = + toPartialBitmask(vceqq_u8(d0, quoteRegister), vceqq_u8(d1, quoteRegister), vceqq_u8(d2, quoteRegister), vceqq_u8(d3, quoteRegister), bitMask); + const uint64x2_t pairedMasks = vreinterpretq_u64_u8(vpaddq_u8(backslashPartial, quotesPartial)); + const uint64_t backslashLocal = vgetq_lane_u64(pairedMasks, 0); + const uint64_t quotesLocal = vgetq_lane_u64(pairedMasks, 1); + const uint64_t escaped = nextEscapeAndTerminalCode(backslashLocal); + const uint64_t quotes = (quotesLocal & ~escaped); + rope_block::escaped = escaped; + rope_block::quotes = quotes; + return quotes ? finishNextInString() : finishNextNoInString(); + } + + JSONIFIER_INLINE static uint8x16_t toPartialBitmask(const uint8x16_t match0, const uint8x16_t match1, const uint8x16_t match2, const uint8x16_t match3, + const uint8x16_t bitMask) noexcept { + const uint8x16_t sum0 = vpaddq_u8(vandq_u8(match0, bitMask), vandq_u8(match1, bitMask)); + const uint8x16_t sum1 = vpaddq_u8(vandq_u8(match2, bitMask), vandq_u8(match3, bitMask)); + return vpaddq_u8(sum0, sum1); + } + + template JSONIFIER_INLINE void nextScalar(const pod_simd_array_t in01, + const simd_register_t bsRegister, const simd_register_t quoteRegister) noexcept { + if constexpr (registerCount == simdRegistersPerBlock) { + next(in01, bsRegister, quoteRegister); + } else { + const uint64_t escaped = nextEscapeAndTerminalCode(pod_cmp_eq_op::impl(in01, bsRegister)); + const uint64_t quotes = (pod_cmp_eq_op::impl(in01, quoteRegister) & ~escaped); + rope_block::escaped = escaped; + rope_block::quotes = quotes; + return quotes ? finishNextInString() : finishNextNoInString(); + } + } + + JSONIFIER_INLINE uint64_t nextEscapeAndTerminalCode(const uint64_t backslashLocal) noexcept { + if (!backslashLocal) { + const uint64_t escaped = nextIsEscaped; + nextIsEscaped = 0; + return escaped; + } + const uint64_t escapeAndTerminalCode = nextEscapeAndTerminalCodeImpl(backslashLocal & ~nextIsEscaped); + const uint64_t escaped = escapeAndTerminalCode ^ (backslashLocal | nextIsEscaped); + nextIsEscaped = (escapeAndTerminalCode & backslashLocal) >> 63; + return escaped; + } + + JSONIFIER_INLINE uint64_t nextEscapeAndTerminalCodeImpl(const uint64_t potentialEscape) noexcept { + static constexpr uint64_t oddBits{ 0xAAAAAAAAAAAAAAAAULL }; + const uint64_t maybeEscaped = potentialEscape << 1; + const uint64_t maybeEscapedAndOddBits = maybeEscaped | oddBits; + const uint64_t evenSeriesCodesAndOddBits = maybeEscapedAndOddBits - potentialEscape; + return evenSeriesCodesAndOddBits ^ oddBits; + } + + JSONIFIER_INLINE uint64_t followsNonquoteScalar(const uint64_t nonquoteScalar) noexcept { + const uint64_t shifted = (nonquoteScalar << 1) | prevScalar; + prevScalar = nonquoteScalar >> 63; + return shifted; + } + + JSONIFIER_INLINE void finishNextInString() noexcept { + const uint64_t inString = prefix_xor_op::impl(rope_block::quotes) ^ prevInString; + prevInString = static_cast(static_cast(inString) >> 63); + rope_block::inString = inString; + } + + JSONIFIER_INLINE void finishNextNoInString() noexcept { + rope_block::inString = prevInString; + } + + uint64_t nextIsEscaped{}; + uint64_t prevInString{}; + uint64_t prevScalar{}; + }; + + struct tape_writer_op { + JSONIFIER_INLINE static uint32_t extractIndex(const uint64_t base, const uint64_t bits) noexcept { + return static_cast(countrZero(bits) + base); + } + + JSONIFIER_INLINE static uint64_t correctedPopcount(const uint64_t bits) noexcept { + return static_cast(popcount(bits)); + } + + JSONIFIER_INLINE static uint64_t advance(const uint64_t bits) noexcept { + return blsr(bits); + } + }; + + template consteval array generateWhitespaceArrayNeon() noexcept { + constexpr const uint8_t values[]{ 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0xFFu, 0xFFu, 0x00u, 0x00u, 0xFFu, 0x00u, 0x00u }; + array returnValues{}; + for (uint64_t x = 0; x < size; ++x) { + returnValues[x] = values[x % 16]; + } + return returnValues; + }; + + template alignas(simdBytesPerRegister) inline constexpr array whitespaceArray{ generateWhitespaceArrayNeon() }; + + template consteval array generateOpArrayNeon() noexcept { + constexpr const uint8_t values[]{ 0xFFu, 0x00u, uint_c_v<','>, uint_c_v<':'>, 0x00u, uint_c_v<'['>, uint_c_v<']'>, uint_c_v<'{'>, uint_c_v<'}'>, 0x00u, 0x00u, 0x00u, + 0x00u, 0x00u, 0x00u, 0x00u }; + array returnValues{}; + for (uint64_t x = 0; x < size; ++x) { + returnValues[x] = values[x % 16]; + } + return returnValues; + }; + + template alignas(simdBytesPerRegister) inline constexpr array opArray{ generateOpArrayNeon() }; + +#endif + +} diff --git a/deps/jsonifier/include/jsonifier-incl/simd/simd_config.hpp b/deps/jsonifier/include/jsonifier-incl/simd/simd_config.hpp new file mode 100644 index 000000000000..29da15bb9d16 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/simd/simd_config.hpp @@ -0,0 +1,145 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/simd/simd_config.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::internal { + + using read_structural_index_ptr = const uint32_t* __restrict; + using read_buffer_ptr = const uint8_t* __restrict; + using write_structural_index_ptr = uint32_t* __restrict; + using write_buffer_ptr = uint8_t* __restrict; + +#if JSONIFIER_AVX512 + inline constexpr uint64_t simdBlocksPerStep{ 8 }; +#elif JSONIFIER_AVX2 + inline constexpr uint64_t simdTapeStep{ 4 }; + #if JSONIFIER_COMPILER_CLANG + inline constexpr uint64_t simdBlocksPerStep{ 4 }; + #else + inline constexpr uint64_t simdBlocksPerStep{ 8 }; + #endif +#elif JSONIFIER_NEON + inline constexpr uint64_t simdTapeStep{ 8 }; + inline constexpr uint64_t simdBlocksPerStep{ 4 }; +#else + inline constexpr uint64_t simdTapeStep{ 8 }; + inline constexpr uint64_t simdBlocksPerStep{ 8 }; +#endif + +#if JSONIFIER_ANY_AVX + + using jsonifier_simd_int_128 = __m128i; + using jsonifier_simd_int_256 = __m256i; + using jsonifier_simd_int_512 = __m512i; + + #if JSONIFIER_AVX512 + inline constexpr const char* cpuArchNameInternal{ "AVX512" }; + using jsonifier_simd_int_t = jsonifier_simd_int_512; + #elif JSONIFIER_AVX2 + inline constexpr const char* cpuArchNameInternal{ "AVX2" }; + using jsonifier_simd_int_t = jsonifier_simd_int_256; + #elif JSONIFIER_AVX + inline constexpr const char* cpuArchNameInternal{ "AVX" }; + using jsonifier_simd_int_t = jsonifier_simd_int_128; + #endif + +#elif JSONIFIER_SVE2 + + static_assert(JSONIFIER_SVE2_VECTOR_BITS == 128, "Jsonifier's SVE2 path is only implemented for a 128-bit vector length."); + + using jsonifier_simd_int_128 = svuint8_t __attribute__((arm_sve_vector_bits(JSONIFIER_SVE2_VECTOR_BITS))); + using jsonifier_simd_int_256 = uint32_t; + using jsonifier_simd_int_512 = uint64_t; + + inline constexpr const char* cpuArchNameInternal{ "SVE2" }; + using jsonifier_simd_int_t = jsonifier_simd_int_128; + +#elif JSONIFIER_NEON + + using jsonifier_simd_int_128 = uint8x16_t; + using jsonifier_simd_int_256 = uint32_t; + using jsonifier_simd_int_512 = uint64_t; + + inline constexpr const char* cpuArchNameInternal{ "NEON" }; + using jsonifier_simd_int_t = jsonifier_simd_int_128; + +#else + + using jsonifier_simd_int_128 = internal::simd::simd_x; + using jsonifier_simd_int_256 = uint32_t; + using jsonifier_simd_int_512 = uint64_t; + + inline constexpr const char* cpuArchNameInternal{ "FALLBACK" }; + using jsonifier_simd_int_t = jsonifier_simd_int_128; + +#endif + + inline constexpr uint64_t simdBytesPerRegister{ internal::cpu_properties::getValue(internal::cpu_property_types::alignment) }; + inline constexpr uint64_t simdRegistersPerBlock{ 64 / simdBytesPerRegister }; + inline constexpr uint64_t simdBytesPerBlock{ 64 }; + inline constexpr uint64_t simdBytesPerStep = simdBlocksPerStep * simdBytesPerBlock; + +#if JSONIFIER_AVX2 && !JSONIFIER_AVX512 && JSONIFIER_COMPILER_GCC + inline constexpr uint64_t podBlocksPerStep{ 4 }; +#else + inline constexpr uint64_t podBlocksPerStep{ simdBlocksPerStep }; +#endif + inline constexpr uint64_t podBytesPerStep = podBlocksPerStep * simdBytesPerBlock; + + static_assert(simdBytesPerRegister == sizeof(jsonifier_simd_int_t), + "simdBytesPerRegister disagrees with the actual register width; simdRegistersPerBlock and every bitmask collapse depend on these matching."); + static_assert(simdBytesPerBlock % simdBytesPerRegister == 0, "Register width must evenly divide the 64-byte block."); + + template struct simd_register_selector { + using type = void; + }; + + template<> struct simd_register_selector<16> { + using type = jsonifier_simd_int_128; + }; + + template<> struct simd_register_selector<32> { + using type = jsonifier_simd_int_256; + }; + + template<> struct simd_register_selector<64> { + using type = jsonifier_simd_int_512; + }; + + template using simd_register_t = typename simd_register_selector::type; + + template struct simd_register_array { + using simd_type = simd_register_t; + + template JSONIFIER_INLINE void set(const simd_type value) noexcept { + static_assert(indexNew < registerCount, "simd_register_array::set index out of range."); + values[indexNew] = value; + } + + template JSONIFIER_INLINE simd_type get() const noexcept { + static_assert(indexNew < registerCount, "simd_register_array::get index out of range."); + return values[indexNew]; + } + + alignas(registerBytes) simd_type values[registerCount]; + }; + + template using simd_array = simd_register_array; + + using simd_array_t = simd_array; + + template using pod_simd_array_t = simd_register_array; +} + +namespace jsonifier { + + inline constexpr const char* cpuArchName{ internal::cpuArchNameInternal }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/simd/simd_x.hpp b/deps/jsonifier/include/jsonifier-incl/simd/simd_x.hpp new file mode 100644 index 000000000000..bfe975dff0e2 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/simd/simd_x.hpp @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/simd/simd_x.hpp + */ +#pragma once + +#include + +namespace jsonifier::internal::simd { + + struct simd_x { + public: + static constexpr uint64_t sixtyFourPer{ 16ULL / sizeof(uint64_t) }; + static constexpr uint64_t thirtyTwoPer{ 16ULL / sizeof(uint32_t) }; + static constexpr uint64_t sixteenPer{ 16ULL / sizeof(uint16_t) }; + static constexpr uint64_t eightPer{ 16ULL / sizeof(uint8_t) }; + + union alignas(16) storage_type { + uint64_t xUint64[sixtyFourPer]; + uint32_t xUint32[thirtyTwoPer]; + uint16_t xUint16[sixteenPer]; + uint8_t xUint8[eightPer]; + }; + + JSONIFIER_INLINE simd_x() noexcept = default; + JSONIFIER_INLINE simd_x& operator=(const simd_x&) noexcept = default; + JSONIFIER_INLINE simd_x(const simd_x&) noexcept = default; + + storage_type values; + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/simd/sve2.hpp b/deps/jsonifier/include/jsonifier-incl/simd/sve2.hpp new file mode 100644 index 000000000000..001d4bae948b --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/simd/sve2.hpp @@ -0,0 +1,132 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/simd/sve2.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::internal::simd { + +#if JSONIFIER_SVE2 + + template [[maybe_unused]] JSONIFIER_INLINE jsonifier_simd_int_t opSrLi(simd_int_t01 value) noexcept { + return svlsr_n_u8_x(svptrue_b8(), value, alignment); + } + + template [[maybe_unused]] JSONIFIER_INLINE void storeU(simd_int_type_new value, void* storageLocation) noexcept { + svst1_u8(svptrue_b8(), static_cast(storageLocation), value); + } + + template [[maybe_unused]] JSONIFIER_INLINE void store(simd_int_type_new value, void* storageLocation) noexcept { + svst1_u8(svptrue_b8(), static_cast(storageLocation), value); + } + + template [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValuesU(const void* str) noexcept { + return svld1_u8(svptrue_b8(), static_cast(str)); + } + + template [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValues(const void* str) noexcept { + return svld1_u8(svptrue_b8(), static_cast(str)); + } + + template [[maybe_unused]] JSONIFIER_INLINE uint64_t opBitMaskRaw(simd_int_t01 value) noexcept { + static_assert(JSONIFIER_SVE2_VECTOR_BITS == 128, "opBitMaskRaw's nibble-per-byte packing only fits a uint64_t at 128-bit VL."); + const jsonifier_simd_int_t narrowed = svshrnb_n_u16(svreinterpret_u16_u8(value), 4); + const jsonifier_simd_int_t packed = svuzp1_u8(narrowed, narrowed); + return svlastb_u64(svptrue_pat_b64(SV_VL1), svreinterpret_u64_u8(packed)); + } + + template [[maybe_unused]] JSONIFIER_INLINE uint16_t opBitMask(simd_int_t01 value) noexcept { + static_assert(JSONIFIER_SVE2_VECTOR_BITS == 128, "opBitMask returns uint16_t and therefore only covers a 128-bit vector."); + const uint8x16_t masked = vandq_u8(svget_neonq_u8(value), vreinterpretq_u8_u64(vdupq_n_u64(0x8040201008040201ULL))); + const uint64x2_t summed = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(masked))); + return static_cast(vgetq_lane_u64(summed, 0) | (vgetq_lane_u64(summed, 1) << 8)); + } + + template [[maybe_unused]] JSONIFIER_INLINE uint64_t postCmpTzcntUnsafe(value_type value) noexcept { + return countrZeroUnsafe(value) >> 2; + } + + template [[maybe_unused]] JSONIFIER_INLINE bool opTest(simd_int_t01 value) noexcept { + const auto pg = svptrue_b8(); + return !svptest_any(pg, svcmpne_n_u8(pg, value, 0)); + } + + template + [[maybe_unused]] JSONIFIER_INLINE jsonifier_simd_int_t opPrev(simd_int_t01 current, simd_int_t02 previous) noexcept { + return svext_u8(previous, current, shift); + } + + template [[maybe_unused]] JSONIFIER_INLINE uint64_t postCmpTzcnt(value_type value) noexcept { + return countrZero(value) >> 2; + } + + template + [[maybe_unused]] JSONIFIER_INLINE jsonifier_simd_int_t opCmpEqRaw(simd_int_t01 value, simd_int_t02 other) noexcept { + return svdup_n_u8_z(svcmpeq_u8(svptrue_b8(), value, other), 0xFF); + } + + template + [[maybe_unused]] JSONIFIER_INLINE jsonifier_simd_int_t opCmpLtRaw(simd_int_t01 value, simd_int_t02 other) noexcept { + return svdup_n_u8_z(svcmplt_u8(svptrue_b8(), value, other), 0xFF); + } + + template + [[maybe_unused]] JSONIFIER_INLINE jsonifier_simd_int_t opShuffle(simd_int_t01 value, simd_int_t02 other) noexcept { + const jsonifier_simd_int_t bitMask = svreinterpret_u8_u8(svdup_n_u8(0x0F)); + return svtbl_u8(value, svand_u8_x(svptrue_b8(), other, bitMask)); + } + + template + [[maybe_unused]] JSONIFIER_INLINE jsonifier_simd_int_t opSubs(simd_int_t01 value, simd_int_t02 other) noexcept { + return svqsub_u8_x(svptrue_b8(), value, other); + } + + template + [[maybe_unused]] JSONIFIER_INLINE jsonifier_simd_int_t opXor(simd_int_t01 value, simd_int_t02 other) noexcept { + return sveor_u8_x(svptrue_b8(), value, other); + } + + template + [[maybe_unused]] JSONIFIER_INLINE jsonifier_simd_int_t opAnd(simd_int_t01 value, simd_int_t02 other) noexcept { + return svand_u8_x(svptrue_b8(), value, other); + } + + template + [[maybe_unused]] JSONIFIER_INLINE jsonifier_simd_int_t opOr(simd_int_t01 value, simd_int_t02 other) noexcept { + return svorr_u8_x(svptrue_b8(), value, other); + } + + template JSONIFIER_INLINE simd_int_t01 orAll(simd_array chunks) noexcept { + return opOr(chunks.template get<0>(), opOr(chunks.template get<1>(), opOr(chunks.template get<2>(), chunks.template get<3>()))); + } + + template + [[maybe_unused]] JSONIFIER_INLINE uint64_t opCmpEq(simd_int_t01 value, simd_int_t02 other) noexcept { + return opBitMaskRaw(opCmpEqRaw(value, other)); + } + + template JSONIFIER_INLINE bool isAscii(simd_int_t input) noexcept { + const auto pg = svptrue_b8(); + return !svptest_any(pg, svcmpge_n_u8(pg, input, 0x80)); + } + + template + requires(sizeof(char_t) == 8) + [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValue(char_t str) noexcept { + return svreinterpret_u8_u64(svdup_n_u64(static_cast(str))); + } + + template + requires(sizeof(char_t) == 1) + [[maybe_unused]] JSONIFIER_INLINE simd_int_type_new gatherValue(char_t str) noexcept { + return svdup_n_u8(static_cast(str)); + } + +#endif + +} diff --git a/deps/jsonifier/include/jsonifier-incl/simd/sve2_stage1.hpp b/deps/jsonifier/include/jsonifier-incl/simd/sve2_stage1.hpp new file mode 100644 index 000000000000..5bf5127ec679 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/simd/sve2_stage1.hpp @@ -0,0 +1,230 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/simd/sve2_stage1.hpp + */ +// Sampled from Dr. Lemire's library, simdjson: https://github.com/simdjson/simdjson +#pragma once + +#include + +namespace jsonifier::internal::simd { + +#if JSONIFIER_SVE2 + + static_assert(JSONIFIER_SVE2_VECTOR_BITS == 128, + "This block collapses simdRegistersPerBlock registers into a single uint64_t bitmask, and SVE2 ADDP is segment-wise; both hold only at 128-bit VL."); + + inline constexpr array shiftAmounts{ [] { + array returnValue{}; + for (uint64_t x = 0; x < simdRegistersPerBlock; ++x) { + returnValue[x] = simdBytesPerRegister * x; + } + return returnValue; + }() }; + + consteval uint64_t getShiftAmount(const uint64_t index) noexcept { + return shiftAmounts[index % shiftAmounts.size()]; + } + + alignas(16) inline constexpr uint8_t sve2BitMaskPattern[16]{ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 }; + + struct prefix_xor_op { + JSONIFIER_INLINE static uint64_t impl(uint64_t bitmask) noexcept { + const poly64_t allOnes = static_cast(~0ull); + const poly128_t result = vmull_p64(static_cast(bitmask), allOnes); + return static_cast(vgetq_lane_u64(vreinterpretq_u64_p128(result), 0)); + } + }; + + JSONIFIER_INLINE jsonifier_simd_int_t sve2OpChars() noexcept { + alignas(16) static constexpr uint8_t opPattern[16]{ uint_c_v<','>, uint_c_v<':'>, uint_c_v<'['>, uint_c_v<']'>, uint_c_v<'{'>, uint_c_v<'}'>, uint_c_v<','>, uint_c_v<':'>, + uint_c_v<'['>, uint_c_v<']'>, uint_c_v<'{'>, uint_c_v<'}'>, uint_c_v<','>, uint_c_v<':'>, uint_c_v<'['>, uint_c_v<']'> }; + return svld1_u8(svptrue_b8(), opPattern); + } + + JSONIFIER_INLINE jsonifier_simd_int_t sve2WsChars() noexcept { + alignas(16) static constexpr uint8_t wsPattern[16]{ uint_c_v<' '>, uint_c_v<'\t'>, uint_c_v<'\n'>, uint_c_v<'\r'>, uint_c_v<' '>, uint_c_v<'\t'>, uint_c_v<'\n'>, + uint_c_v<'\r'>, uint_c_v<' '>, uint_c_v<'\t'>, uint_c_v<'\n'>, uint_c_v<'\r'>, uint_c_v<' '>, uint_c_v<'\t'>, uint_c_v<'\n'>, uint_c_v<'\r'> }; + return svld1_u8(svptrue_b8(), wsPattern); + } + + JSONIFIER_INLINE uint64_t sve2CollapseMasked(const jsonifier_simd_int_t masked0, const jsonifier_simd_int_t masked1, const jsonifier_simd_int_t masked2, + const jsonifier_simd_int_t masked3) noexcept { + uint8x16_t sum0 = vpaddq_u8(svget_neonq_u8(masked0), svget_neonq_u8(masked1)); + const uint8x16_t sum1 = vpaddq_u8(svget_neonq_u8(masked2), svget_neonq_u8(masked3)); + sum0 = vpaddq_u8(sum0, sum1); + sum0 = vpaddq_u8(sum0, sum0); + return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0); + } + + JSONIFIER_INLINE uint64_t sve2CollapseToBitmask(const jsonifier_simd_int_t match0, const jsonifier_simd_int_t match1, const jsonifier_simd_int_t match2, + const jsonifier_simd_int_t match3, const jsonifier_simd_int_t bitMask) noexcept { + const auto pg = svptrue_b8(); + return sve2CollapseMasked(svand_u8_x(pg, match0, bitMask), svand_u8_x(pg, match1, bitMask), svand_u8_x(pg, match2, bitMask), svand_u8_x(pg, match3, bitMask)); + } + + JSONIFIER_INLINE jsonifier_simd_int_t sve2BitMask() noexcept { + return svreinterpret_u8_u64(svdup_n_u64(0x8040201008040201ULL)); + } + + struct ws_collector { + JSONIFIER_INLINE static uint64_t impl(const simd_array_t in01, const jsonifier_simd_int_t) noexcept { + const auto pg = svptrue_b8(); + const auto bitMask = sve2BitMask(); + const auto wsChars = sve2WsChars(); + return sve2CollapseMasked(svand_u8_z(svmatch_u8(pg, in01.template get<0>(), wsChars), bitMask, bitMask), + svand_u8_z(svmatch_u8(pg, in01.template get<1>(), wsChars), bitMask, bitMask), svand_u8_z(svmatch_u8(pg, in01.template get<2>(), wsChars), bitMask, bitMask), + svand_u8_z(svmatch_u8(pg, in01.template get<3>(), wsChars), bitMask, bitMask)); + } + }; + + struct op_collector { + JSONIFIER_INLINE static uint64_t impl(const simd_array_t in01, const jsonifier_simd_int_t, const jsonifier_simd_int_t) noexcept { + const auto pg = svptrue_b8(); + const auto bitMask = sve2BitMask(); + const auto opChars = sve2OpChars(); + return sve2CollapseMasked(svand_u8_z(svmatch_u8(pg, in01.template get<0>(), opChars), bitMask, bitMask), + svand_u8_z(svmatch_u8(pg, in01.template get<1>(), opChars), bitMask, bitMask), svand_u8_z(svmatch_u8(pg, in01.template get<2>(), opChars), bitMask, bitMask), + svand_u8_z(svmatch_u8(pg, in01.template get<3>(), opChars), bitMask, bitMask)); + } + }; + + struct unescaped_collector { + JSONIFIER_INLINE static uint64_t impl(const simd_array_t in01) noexcept { + const auto pg = svptrue_b8(); + const auto bitMask = sve2BitMask(); + return sve2CollapseMasked(svand_u8_x(pg, in01.template get<0>(), bitMask), svand_u8_x(pg, in01.template get<1>(), bitMask), + svand_u8_x(pg, in01.template get<2>(), bitMask), svand_u8_x(pg, in01.template get<3>(), bitMask)); + } + }; + + template struct rope_detector : rope_block { + JSONIFIER_INLINE void next(const simd_array_t in01, const jsonifier_simd_int_t bsRegister, const jsonifier_simd_int_t quoteRegister) noexcept { + const auto pg = svptrue_b8(); + const auto bitMask = sve2BitMask(); + const jsonifier_simd_int_t d0 = in01.template get<0>(); + const jsonifier_simd_int_t d1 = in01.template get<1>(); + const jsonifier_simd_int_t d2 = in01.template get<2>(); + const jsonifier_simd_int_t d3 = in01.template get<3>(); + const auto eqBs0 = svcmpeq_u8(pg, d0, bsRegister); + const auto eqQ0 = svcmpeq_u8(pg, d0, quoteRegister); + const auto eqBs1 = svcmpeq_u8(pg, d1, bsRegister); + const auto eqQ1 = svcmpeq_u8(pg, d1, quoteRegister); + const auto eqBs2 = svcmpeq_u8(pg, d2, bsRegister); + const auto eqQ2 = svcmpeq_u8(pg, d2, quoteRegister); + const auto eqBs3 = svcmpeq_u8(pg, d3, bsRegister); + const auto eqQ3 = svcmpeq_u8(pg, d3, quoteRegister); + const uint64_t backslashLocal = sve2CollapseMasked(svand_u8_z(eqBs0, bitMask, bitMask), svand_u8_z(eqBs1, bitMask, bitMask), svand_u8_z(eqBs2, bitMask, bitMask), + svand_u8_z(eqBs3, bitMask, bitMask)); + const uint64_t quotesLocal = + sve2CollapseMasked(svand_u8_z(eqQ0, bitMask, bitMask), svand_u8_z(eqQ1, bitMask, bitMask), svand_u8_z(eqQ2, bitMask, bitMask), svand_u8_z(eqQ3, bitMask, bitMask)); + const uint64_t escaped = nextEscapeAndTerminalCode(backslashLocal); + const uint64_t quotes = (quotesLocal & ~escaped); + rope_block::escaped = escaped; + rope_block::quotes = quotes; + return quotes ? finishNext() : finishNextNoInString(); + } + + JSONIFIER_INLINE static uint64_t toBitmask(const jsonifier_simd_int_t masked0, const jsonifier_simd_int_t masked1, const jsonifier_simd_int_t masked2, + const jsonifier_simd_int_t masked3) noexcept { + return sve2CollapseMasked(masked0, masked1, masked2, masked3); + } + + template JSONIFIER_INLINE void nextScalar(const pod_simd_array_t in01, + const simd_register_t bsRegister, const simd_register_t quoteRegister) noexcept { + next(in01, bsRegister, quoteRegister); + } + + JSONIFIER_INLINE uint64_t nextEscapeAndTerminalCode(const uint64_t backslashLocal) noexcept { + if (!backslashLocal) { + const uint64_t escaped = nextIsEscaped; + nextIsEscaped = 0; + return escaped; + } + const uint64_t escapeAndTerminalCode = nextEscapeAndTerminalCodeImpl(backslashLocal & ~nextIsEscaped); + const uint64_t escaped = escapeAndTerminalCode ^ (backslashLocal | nextIsEscaped); + nextIsEscaped = (escapeAndTerminalCode & backslashLocal) >> 63; + return escaped; + } + + JSONIFIER_INLINE uint64_t nextEscapeAndTerminalCodeImpl(const uint64_t potentialEscape) noexcept { + static constexpr uint64_t oddBits{ 0xAAAAAAAAAAAAAAAAULL }; + const uint64_t maybeEscaped = potentialEscape << 1; + const uint64_t maybeEscapedAndOddBits = maybeEscaped | oddBits; + const uint64_t evenSeriesCodesAndOddBits = maybeEscapedAndOddBits - potentialEscape; + return evenSeriesCodesAndOddBits ^ oddBits; + } + + JSONIFIER_INLINE uint64_t followsNonquoteScalar(const uint64_t nonquoteScalar) noexcept { + const uint64_t shifted = (nonquoteScalar << 1) | prevScalar; + prevScalar = nonquoteScalar >> 63; + return shifted; + } + + JSONIFIER_INLINE void finishNext() noexcept { + const uint64_t inString = prefix_xor_op::impl(rope_block::quotes) ^ prevInString; + prevInString = static_cast(static_cast(inString) >> 63); + rope_block::inString = inString; + } + + JSONIFIER_INLINE void finishNextNoInString() noexcept { + rope_block::inString = prevInString; + } + + JSONIFIER_INLINE void reset() noexcept { + nextIsEscaped = 0; + prevInString = 0; + prevScalar = 0; + } + + uint64_t nextIsEscaped{}; + uint64_t prevInString{}; + uint64_t prevScalar{}; + }; + + template using pod_ws_collector = ws_collector; + + template using scalar_op_collector = op_collector; + + struct tape_writer_op { + JSONIFIER_INLINE static uint32_t extractIndex(const uint64_t base, const uint64_t bits) noexcept { + return static_cast(countrZero(bits) + base); + } + + JSONIFIER_INLINE static uint64_t correctedPopcount(const uint64_t bits) noexcept { + return static_cast(popcount(bits)); + } + + JSONIFIER_INLINE static uint64_t advance(const uint64_t bits) noexcept { + return blsr(bits); + } + }; + + template consteval array generateWhitespaceArraySve2() noexcept { + constexpr const uint8_t values[]{ 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0xFFu, 0xFFu, 0x00u, 0x00u, 0xFFu, 0x00u, 0x00u }; + array returnValues{}; + for (uint64_t x = 0; x < size; ++x) { + returnValues[x] = values[x % 16]; + } + return returnValues; + }; + + template alignas(simdBytesPerRegister) inline constexpr array whitespaceArray{ generateWhitespaceArraySve2() }; + + template consteval array generateOpArraySve2() noexcept { + constexpr const uint8_t values[]{ 0xFFu, 0x00u, uint_c_v<','>, uint_c_v<':'>, 0x00u, uint_c_v<'['>, uint_c_v<']'>, uint_c_v<'{'>, uint_c_v<'}'>, 0x00u, 0x00u, 0x00u, + 0x00u, 0x00u, 0x00u, 0x00u }; + array returnValues{}; + for (uint64_t x = 0; x < size; ++x) { + returnValues[x] = values[x % 16]; + } + return returnValues; + }; + + template alignas(simdBytesPerRegister) inline constexpr array opArray{ generateOpArraySve2() }; + +#endif + +} diff --git a/deps/jsonifier/include/jsonifier-incl/simd/utf8_validation.hpp b/deps/jsonifier/include/jsonifier-incl/simd/utf8_validation.hpp new file mode 100644 index 000000000000..81f18f9cd73c --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/simd/utf8_validation.hpp @@ -0,0 +1,360 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/simd/utf8_validation.hpp + */ +// Sampled from Dr. Lemire's library, simdjson: https://github.com/simdjson/simdjson +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace jsonifier::internal { + + inline constexpr uint8_t tooShort = 1 << 0; + inline constexpr uint8_t tooLong = 1 << 1; + inline constexpr uint8_t overLong3 = 1 << 2; + inline constexpr uint8_t tooLarge = 1 << 3; + inline constexpr uint8_t surrogate = 1 << 4; + inline constexpr uint8_t overLong2 = 1 << 5; + inline constexpr uint8_t twoConts = 1 << 7; + inline constexpr uint8_t tooLarge1000 = 1 << 6; + inline constexpr uint8_t overLong4 = 1 << 6; + inline constexpr uint8_t carry = tooShort | tooLong | twoConts; + + using namespace simd; + + consteval array genByte1HighTable() noexcept { + constexpr array raw{ { tooLong, tooLong, tooLong, tooLong, tooLong, tooLong, tooLong, tooLong, twoConts, twoConts, twoConts, twoConts, + tooShort | overLong2, tooShort, tooShort | overLong3 | surrogate, tooShort | tooLarge | tooLarge1000 | overLong4 } }; + array returnValue{}; + for (uint64_t x = 0; x < simdBytesPerRegister; ++x) { + returnValue[x] = raw[x % raw.size()]; + } + return returnValue; + } + + alignas(simdBytesPerRegister) inline constexpr auto byte1HighTableRaw{ genByte1HighTable() }; + + inline constexpr read_buffer_ptr byte1HighTable{ byte1HighTableRaw.data() }; + + consteval array genByte1LowTable() noexcept { + constexpr array raw{ { carry | overLong3 | overLong2 | overLong4, carry | overLong2, carry, carry, carry | tooLarge, carry | tooLarge | tooLarge1000, + carry | tooLarge | tooLarge1000, carry | tooLarge | tooLarge1000, carry | tooLarge | tooLarge1000, carry | tooLarge | tooLarge1000, carry | tooLarge | tooLarge1000, + carry | tooLarge | tooLarge1000, carry | tooLarge | tooLarge1000, carry | tooLarge | tooLarge1000 | surrogate, carry | tooLarge | tooLarge1000, + carry | tooLarge | tooLarge1000 } }; + array returnValue{}; + for (uint64_t x = 0; x < simdBytesPerRegister; ++x) { + returnValue[x] = raw[x % raw.size()]; + } + return returnValue; + } + + alignas(simdBytesPerRegister) inline constexpr auto byte1LowTableRaw{ genByte1LowTable() }; + + inline constexpr read_buffer_ptr byte1LowTable{ byte1LowTableRaw.data() }; + + consteval array genByte2HighTable() noexcept { + constexpr array raw{ { tooShort, tooShort, tooShort, tooShort, tooShort, tooShort, tooShort, tooShort, + tooLong | overLong2 | twoConts | overLong3 | tooLarge1000 | overLong4, tooLong | overLong2 | twoConts | overLong3 | tooLarge, + tooLong | overLong2 | twoConts | surrogate | tooLarge, tooLong | overLong2 | twoConts | surrogate | tooLarge, tooShort, tooShort, tooShort, tooShort } }; + array returnValue{}; + for (uint64_t x = 0; x < simdBytesPerRegister; ++x) { + returnValue[x] = raw[x % raw.size()]; + } + return returnValue; + } + + alignas(simdBytesPerRegister) inline constexpr auto byte2HighTableRaw{ genByte2HighTable() }; + + inline constexpr read_buffer_ptr byte2HighTable{ byte2HighTableRaw.data() }; + + alignas(simdBytesPerRegister) inline constexpr uint8_t isIncompleteMax[64]{ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, static_cast(0xF0u - 1), static_cast(0xE0u - 1), static_cast(0xC0u - 1) }; + + template struct chunk_loader; + + template struct chunk_loader> { + JSONIFIER_INLINE static simd_array_t impl(read_buffer_ptr src) noexcept { + simd_array_t returnValues; + (returnValues.template set(gatherValuesU(std::bit_cast(src + indices * simdBytesPerRegister))), + ...); + return returnValues; + } + }; + + template struct step_checker; + + template struct step_checker> { + JSONIFIER_INLINE void impl(read_buffer_ptr srcNew) noexcept { + (static_cast(this)->checkStepImpl(srcNew + indices * simdBytesPerBlock), ...); + } + }; + + struct utf8_checker : public step_checker> { + using step_checker_type = step_checker>; + template struct chunk_processor; + + template struct chunk_processor> { + template JSONIFIER_INLINE static void impl(utf8_checker& __restrict c, simd_array_t chunks, jsonifier_simd_int_t& __restrict prev) noexcept { + c.checkChunk(chunks.template get(), prev), prev = chunks.template get(); + } + + JSONIFIER_INLINE static void impl(utf8_checker& __restrict c, simd_array_t chunks, jsonifier_simd_int_t& __restrict prev) noexcept { + (impl(c, chunks, prev), ...); + } + }; + + JSONIFIER_INLINE utf8_checker() noexcept = default; + utf8_checker& operator=(utf8_checker&&) = delete; + utf8_checker(utf8_checker&&) = delete; + utf8_checker& operator=(const utf8_checker&) = delete; + utf8_checker(const utf8_checker&) = delete; + + JSONIFIER_INLINE jsonifier_simd_int_t checkMultibyteLengths(jsonifier_simd_int_t input, jsonifier_simd_int_t prev, jsonifier_simd_int_t sc) noexcept { + return opXor(opAnd(mustBe23Continuation(opPrev<14>(input, prev), opPrev<13>(input, prev)), gatherValue(static_cast(0x80u))), sc); + } + + JSONIFIER_INLINE jsonifier_simd_int_t checkSpecialCases(jsonifier_simd_int_t input, jsonifier_simd_int_t p1) noexcept { + const jsonifier_simd_int_t loNibbleMask = gatherValue(static_cast(0x0Fu)); + return opAnd(opAnd(opShuffle(lookupH, opAnd(opSrLi<4>(p1), loNibbleMask)), opShuffle(lookupL, opAnd(p1, loNibbleMask))), + opShuffle(lookup2, opAnd(opSrLi<4>(input), loNibbleMask))); + } + + JSONIFIER_INLINE jsonifier_simd_int_t mustBe23Continuation(jsonifier_simd_int_t p2, jsonifier_simd_int_t p3) noexcept { + return opOr(opSubs(p2, gatherValue(static_cast(0xE0u - 0x80u))), + opSubs(p3, gatherValue(static_cast(0xF0u - 0x80u)))); + } + + JSONIFIER_INLINE void checkChunk(jsonifier_simd_int_t input, jsonifier_simd_int_t prev) noexcept { + error = opOr(error, checkMultibyteLengths(input, prev, checkSpecialCases(input, opPrev<15>(input, prev)))); + } + + JSONIFIER_INLINE jsonifier_simd_int_t checkIncomplete(jsonifier_simd_int_t input) noexcept { + return opSubs(input, gatherValues(isIncompleteMax + (64 - simdBytesPerRegister))); + } + + JSONIFIER_INLINE void checkStepImpl(read_buffer_ptr src) noexcept { + simd_array_t chunks = chunk_loader>::impl(src); + + if (isAscii(simd::orAll(chunks))) { + error = opOr(error, prevIncomplete); + prevInput = chunks.template get(); + prevIncomplete = jsonifier_simd_int_t{}; + return; + } + + chunk_processor>::impl(*this, chunks, prevInput); + + prevInput = chunks.template get(); + prevIncomplete = checkIncomplete(chunks.template get()); + } + + JSONIFIER_INLINE void checkStep(read_buffer_ptr srcNew) noexcept { + step_checker_type::impl(srcNew); + } + + JSONIFIER_INLINE bool errors() noexcept { + return !opTest(opOr(error, prevIncomplete)); + } + + JSONIFIER_INLINE void reset() noexcept { + prevIncomplete = jsonifier_simd_int_t{}; + prevInput = jsonifier_simd_int_t{}; + error = jsonifier_simd_int_t{}; + } + + const jsonifier_simd_int_t lookupH{ gatherValues(std::bit_cast(byte1HighTable)) }; + const jsonifier_simd_int_t lookup2{ gatherValues(std::bit_cast(byte2HighTable)) }; + const jsonifier_simd_int_t lookupL{ gatherValues(std::bit_cast(byte1LowTable)) }; + jsonifier_simd_int_t prevIncomplete; + jsonifier_simd_int_t prevInput; + jsonifier_simd_int_t error; + }; + + inline bool validateUtf8(read_buffer_ptr src, uint64_t len) noexcept { + if (len == 0) { + return true; + } + + utf8_checker checker{}; + checker.reset(); + + uint64_t i = 0; + + for (; i + simdBytesPerStep <= len; i += simdBytesPerStep) { + checker.checkStep(src + i); + } + + if (i < len) { + alignas(simdBytesPerRegister) uint8_t tmp[simdBytesPerStep]; + std::memset(tmp, 0x41, simdBytesPerStep); + jsonifierMemcpyUpTo(tmp, src + i, len - i); + checker.checkStep(tmp); + } + + return !checker.errors(); + } + + struct utf8_validation_state { + JSONIFIER_INLINE utf8_validation_state() noexcept { + reset(); + } + + JSONIFIER_INLINE void reset() noexcept { + prevIncomplete = false; + error = false; + prevBytes[0] = 0; + prevBytes[1] = 0; + prevBytes[2] = 0; + } + + uint8_t prevBytes[3]; + bool prevIncomplete; + bool error; + }; + + template struct utf8_register_validator { + static constexpr uint64_t bytesProcessed = sizeof(typename simd_type_new::type); + using simd_type = typename simd_type_new::type; + using simd_type_alias = simd_type; + + JSONIFIER_INLINE utf8_register_validator(utf8_validation_state& stateNew) noexcept : state{ stateNew } { + error = simd_type{}; + if (!state.prevIncomplete && ((state.prevBytes[0] | state.prevBytes[1] | state.prevBytes[2]) & 0x80u) == 0) [[likely]] { + incompleteRegister = simd_type{}; + prevInput = simd_type{}; + touched = false; + return; + } + alignas(simdBytesPerRegister) uint8_t tmp[bytesProcessed]; + tmp[bytesProcessed - 3] = state.prevBytes[0]; + tmp[bytesProcessed - 2] = state.prevBytes[1]; + tmp[bytesProcessed - 1] = state.prevBytes[2]; + incompleteRegister = state.prevIncomplete ? simd::gatherValue(static_cast(0x80u)) : simd_type{}; + prevInput = simd::gatherValues(std::bit_cast(+tmp)); + touched = true; + } + + utf8_register_validator& operator=(const utf8_register_validator&) = delete; + utf8_register_validator(const utf8_register_validator&) = delete; + + JSONIFIER_INLINE void checkRegister(simd_type input) noexcept { + if (simd::isAscii(input)) { + error = simd::opOr(error, incompleteRegister); + prevInput = input; + incompleteRegister = simd_type{}; + return; + } + touched = true; + const simd_type sc = checkSpecialCases(input, simd::opPrev<15>(input, prevInput)); + error = simd::opOr(error, simd::opXor(simd::opAnd(mustBe23Continuation(simd::opPrev<14>(input, prevInput), simd::opPrev<13>(input, prevInput)), maskNibble01()), sc)); + prevInput = input; + incompleteRegister = simd::opSubs(input, incompleteMax()); + } + + JSONIFIER_INLINE simd_type checkSpecialCases(simd_type input, simd_type p1) noexcept { + return simd::opAnd( + simd::opAnd(simd::opShuffle(lookupH(), simd::opAnd(simd::opSrLi<4>(p1), loNibbleMask())), simd::opShuffle(lookupL(), simd::opAnd(p1, loNibbleMask()))), + simd::opShuffle(lookup2(), simd::opAnd(simd::opSrLi<4>(input), loNibbleMask()))); + } + + JSONIFIER_INLINE void flush() noexcept { + if (!touched) { + return; + } + alignas(simdBytesPerRegister) uint8_t tmp[bytesProcessed]; + simd::store(prevInput, tmp); + state.prevBytes[0] = tmp[bytesProcessed - 3]; + state.prevBytes[1] = tmp[bytesProcessed - 2]; + state.prevBytes[2] = tmp[bytesProcessed - 1]; + state.prevIncomplete = tmp[bytesProcessed - 1] >= 0xC0u || tmp[bytesProcessed - 2] >= 0xE0u || tmp[bytesProcessed - 3] >= 0xF0u; + if (!simd::opTest(error)) { + state.error = true; + } + } + + JSONIFIER_INLINE simd_type mustBe23Continuation(simd_type p2, simd_type p3) noexcept { + return simd::opOr(simd::opSubs(p2, continuationMask01()), simd::opSubs(p3, continuationMask02())); + } + + JSONIFIER_INLINE static simd_type lookupH() noexcept { + return simd::gatherValues(std::bit_cast(byte1HighTable)); + } + + JSONIFIER_INLINE static simd_type lookup2() noexcept { + return simd::gatherValues(std::bit_cast(byte2HighTable)); + } + + JSONIFIER_INLINE static simd_type lookupL() noexcept { + return simd::gatherValues(std::bit_cast(byte1LowTable)); + } + + JSONIFIER_INLINE void checkPartial(const void* __restrict src, uint64_t count) noexcept { + if (count == 0) { + return; + } + alignas(simdBytesPerRegister) uint8_t tmp[bytesProcessed]; + std::memset(tmp, 32, bytesProcessed); + jsonifierMemcpyUpTo(tmp, src, count); + checkRegister(simd::gatherValues(std::bit_cast(+tmp))); + } + + JSONIFIER_INLINE static simd_type incompleteMax() noexcept { + return simd::gatherValues(isIncompleteMax + (64 - bytesProcessed)); + } + + JSONIFIER_INLINE bool errors() noexcept { + return state.error || !simd::opTest(simd::opOr(error, incompleteRegister)); + } + + JSONIFIER_INLINE static simd_type continuationMask01() noexcept { + return simd::gatherValue(static_cast(0xE0u - 0x80u)); + } + + JSONIFIER_INLINE static simd_type continuationMask02() noexcept { + return simd::gatherValue(static_cast(0xF0u - 0x80u)); + } + + JSONIFIER_INLINE static simd_type maskNibble01() noexcept { + return simd::gatherValue(static_cast(0x80u)); + } + + JSONIFIER_INLINE static simd_type loNibbleMask() noexcept { + return simd::gatherValue(static_cast(0x0Fu)); + } + + JSONIFIER_INLINE bool hardErrors() noexcept { + return state.error || !simd::opTest(error); + } + + JSONIFIER_INLINE void reset() noexcept { + state.reset(); + prevInput = simd_type{}; + incompleteRegister = simd_type{}; + error = simd_type{}; + touched = false; + } + + simd_type incompleteRegister; + simd_type prevInput; + simd_type error; + utf8_validation_state& state; + bool touched; + }; + +} + +namespace jsonifier { + + inline bool validateUtf8(internal::read_buffer_ptr src, uint64_t len) noexcept { + return internal::validateUtf8(src, len); + } + +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/byteswap.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/byteswap.hpp new file mode 100644 index 000000000000..661ee05214d4 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/byteswap.hpp @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/byteswap.hpp + */ +#pragma once + +#include +#include +#include + +namespace jsonifier::internal { + + template JSONIFIER_INLINE constexpr value_type byteswap(value_type value) noexcept { + if constexpr (sizeof(value_type) == 1) { + return value; + } else { + using unsigned_type = std::make_unsigned_t; + const unsigned_type bits{ static_cast(value) }; +#if JSONIFIER_COMPILER_GCC || JSONIFIER_COMPILER_CLANG + if constexpr (sizeof(value_type) == 2) { + return static_cast(__builtin_bswap16(bits)); + } else if constexpr (sizeof(value_type) == 4) { + return static_cast(__builtin_bswap32(bits)); + } else { + return static_cast(__builtin_bswap64(bits)); + } +#else + if (!std::is_constant_evaluated()) { + if constexpr (sizeof(value_type) == 2) { + return static_cast(_byteswap_ushort(bits)); + } else if constexpr (sizeof(value_type) == 4) { + return static_cast(_byteswap_ulong(bits)); + } else { + return static_cast(_byteswap_uint64(bits)); + } + } + unsigned_type result{}; + for (uint64_t index = 0; index < sizeof(value_type); ++index) { + result = static_cast((result << 8) | ((bits >> (index * 8)) & 0xFF)); + } + return static_cast(result); +#endif + } + } + +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/compare.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/compare.hpp new file mode 100644 index 000000000000..f31c6606aa6e --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/compare.hpp @@ -0,0 +1,614 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/compare.hpp + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace jsonifier::internal { + + template consteval return_type repeatByte() noexcept { + return static_cast(0x0101010101010101ull) * static_cast(repeat); + } + + template inline constexpr return_type repeatByte(uint8_t repeat) noexcept { + return static_cast(static_cast(0x0101010101010101ull) * static_cast(repeat)); + } + + template struct char_comparison { + static constexpr char value{ valueNewer }; + + JSONIFIER_INLINE static const char_type* memchar(const char_type* data, uint64_t lengthNew) noexcept { +#if JSONIFIER_AVX512 + if (lengthNew >= 64) { + using type_list_element = type_list_element_t<0, simd::avx_list>; + using simd_type = typename type_list_element::type::type; + using integer_type = typename type_list_element::integer_type; + static constexpr uint64_t vectorSize = type_list_element::bytesProcessed; + const simd_type searchValue = simd::gatherValue(static_cast(value)); + integer_type mask; + simd_type chunk; + while (lengthNew >= vectorSize) { + chunk = simd::gatherValuesU(data); + mask = simd::opCmpEq(chunk, searchValue); + if (mask != 0) [[unlikely]] { + data += simd::postCmpTzcntUnsafe(mask); + return data; + } + lengthNew -= vectorSize; + data += vectorSize; + } + } +#endif + +#if JSONIFIER_AVX512 || JSONIFIER_AVX2 + if (lengthNew >= 32) { + using type_list_element = type_list_element_t<1, simd::avx_list>; + using simd_type = typename type_list_element::type::type; + using integer_type = typename type_list_element::integer_type; + static constexpr uint64_t vectorSize = type_list_element::bytesProcessed; + const simd_type searchValue = simd::gatherValue(static_cast(value)); + integer_type mask; + simd_type chunk; + while (lengthNew >= vectorSize) { + chunk = simd::gatherValuesU(data); + mask = simd::opCmpEq(chunk, searchValue); + if (mask != 0) [[unlikely]] { + data += simd::postCmpTzcntUnsafe(mask); + return data; + } + lengthNew -= vectorSize; + data += vectorSize; + } + } +#endif + if (lengthNew >= 16) { + using type_list_element = type_list_element_t<2, simd::avx_list>; + using simd_type = typename type_list_element::type::type; + using integer_type = typename type_list_element::integer_type; + static constexpr uint64_t vectorSize = type_list_element::bytesProcessed; + const simd_type searchValue = simd::gatherValue(static_cast(value)); + integer_type mask; + simd_type chunk; + while (lengthNew >= vectorSize) { + chunk = simd::gatherValuesU(data); + mask = static_cast(simd::opCmpEq(chunk, searchValue)); + if (mask != 0) [[unlikely]] { + data += simd::postCmpTzcntUnsafe(mask); + return data; + } + lengthNew -= vectorSize; + data += vectorSize; + } + } + { + static constexpr uint64_t nBytes{ sizeof(uint64_t) }; + if (lengthNew >= nBytes) { + static constexpr auto valueNew{ repeatByte() }; + static constexpr auto highBits{ repeatByte<0x80, uint64_t>() }; + static constexpr auto lowBits{ repeatByte<0x01, uint64_t>() }; + uint64_t simdValue; + pow2MemcpyWrapper(&simdValue, data); + if constexpr (std::endian::native == std::endian::big) { + simdValue = internal::byteswap(simdValue); + } + const auto chunk = simdValue ^ valueNew; + auto next = ((chunk - lowBits) & ~chunk) & highBits; + if (next) { + next = simd::countrZeroUnsafe(static_cast(next)) >> 3u; + data += next; + return data; + } else { + lengthNew -= nBytes; + data += nBytes; + } + } + } + { + static constexpr uint64_t nBytes{ sizeof(uint32_t) }; + if (lengthNew >= nBytes) { + static constexpr auto valueNew{ repeatByte() }; + static constexpr auto highBits{ repeatByte<0x80, uint32_t>() }; + static constexpr auto lowBits{ repeatByte<0x01, uint32_t>() }; + uint32_t simdValue; + pow2MemcpyWrapper(&simdValue, data); + if constexpr (std::endian::native == std::endian::big) { + simdValue = internal::byteswap(simdValue); + } + const auto chunk = simdValue ^ valueNew; + auto next = ((chunk - lowBits) & ~chunk) & highBits; + if (next) { + next = static_cast(simd::countrZeroUnsafe(static_cast(next)) >> 3u); + data += next; + return data; + } else { + lengthNew -= nBytes; + data += nBytes; + } + } + } + { + static constexpr uint64_t nBytes{ sizeof(uint16_t) }; + if (lengthNew >= nBytes) { + static constexpr auto valueNew{ repeatByte() }; + static constexpr auto highBits{ repeatByte<0x80, uint16_t>() }; + static constexpr auto lowBits{ repeatByte<0x01, uint16_t>() }; + uint16_t simdValue; + pow2MemcpyWrapper(&simdValue, data); + if constexpr (std::endian::native == std::endian::big) { + simdValue = internal::byteswap(simdValue); + } + const auto chunk = simdValue ^ valueNew; + auto next = ((chunk - lowBits) & ~chunk) & highBits; + if (next) { + next = static_cast(simd::countrZeroUnsafe(static_cast(next)) >> 3u); + data += next; + return data; + } else { + lengthNew -= nBytes; + data += nBytes; + } + } + } + + if (lengthNew > 0 && *data == value) { + return data; + } + return static_cast(nullptr); + } + }; + + struct comparison { + template JSONIFIER_INLINE static bool compare(const char_type01* lhs, char_type02* rhs, uint64_t lengthNew) noexcept { +#if JSONIFIER_AVX512 + if (lengthNew >= 64) { + using type_list_element = type_list_element_t<0, simd::avx_list>; + using simd_type = typename type_list_element::type::type; + static constexpr uint64_t vectorSize = type_list_element::bytesProcessed; + static constexpr uint64_t mask = type_list_element::mask; + simd_type value01, value02; + while (lengthNew >= vectorSize) { + value01 = simd::gatherValuesU(lhs); + value02 = simd::gatherValuesU(rhs); + if (simd::opCmpEq(value01, value02) != mask) { + return false; + } + lengthNew -= vectorSize; + lhs += vectorSize; + rhs += vectorSize; + } + } +#endif +#if JSONIFIER_AVX512 || JSONIFIER_AVX2 + if (lengthNew >= 32) { + using type_list_element = type_list_element_t<1, simd::avx_list>; + using simd_type = typename type_list_element::type::type; + static constexpr uint64_t vectorSize = type_list_element::bytesProcessed; + static constexpr uint64_t mask = type_list_element::mask; + simd_type value01, value02; + while (lengthNew >= vectorSize) { + value01 = simd::gatherValuesU(lhs); + value02 = simd::gatherValuesU(rhs); + if (simd::opCmpEq(value01, value02) != mask) { + return false; + } + lengthNew -= vectorSize; + lhs += vectorSize; + rhs += vectorSize; + } + } +#endif + if (lengthNew >= 16) { + using type_list_element = type_list_element_t<2, simd::avx_list>; + using simd_type = typename type_list_element::type::type; + static constexpr uint64_t vectorSize = type_list_element::bytesProcessed; + static constexpr uint64_t mask = type_list_element::mask; + simd_type value01, value02; + while (lengthNew >= vectorSize) { + value01 = simd::gatherValuesU(lhs); + value02 = simd::gatherValuesU(rhs); + if (simd::opCmpEq(value01, value02) != mask) { + return false; + } + lengthNew -= vectorSize; + lhs += vectorSize; + rhs += vectorSize; + } + } + { + static constexpr uint64_t nBytes{ sizeof(uint64_t) }; + if (lengthNew >= nBytes) { + uint64_t v1, v2; + pow2MemcpyWrapper(&v1, lhs); + pow2MemcpyWrapper(&v2, rhs); + if ((v1 ^ v2) != 0) { + return false; + } + lengthNew -= nBytes; + lhs += nBytes; + rhs += nBytes; + } + } + { + static constexpr uint64_t nBytes{ sizeof(uint32_t) }; + if (lengthNew >= nBytes) { + uint32_t v1, v2; + pow2MemcpyWrapper(&v1, lhs); + pow2MemcpyWrapper(&v2, rhs); + if ((v1 ^ v2) != 0) { + return false; + } + lengthNew -= nBytes; + lhs += nBytes; + rhs += nBytes; + } + } + { + static constexpr uint64_t nBytes{ sizeof(uint16_t) }; + if (lengthNew >= nBytes) { + uint16_t v1, v2; + pow2MemcpyWrapper(&v1, lhs); + pow2MemcpyWrapper(&v2, rhs); + if ((v1 ^ v2) != 0) { + return false; + } + lengthNew -= nBytes; + lhs += nBytes; + rhs += nBytes; + } + } + if (lengthNew && *lhs != *rhs) { + return false; + } + return true; + } + }; + + template struct convert_length_to_int { + using type = conditional_t>>>; + }; + + template using convert_length_to_int_t = typename convert_length_to_int::type; + + template inline constexpr uint64_t intLength = [] { + if constexpr (lengthNew == 1ULL) { + return 1ULL; + } else if constexpr (lengthNew == 2ULL) { + return 2ULL; + } else if constexpr (lengthNew <= 4ULL && lengthNew > 2ULL) { + return 4ULL; + } else if constexpr (lengthNew <= 8ULL && lengthNew > 4ULL) { + return 8ULL; + } else if constexpr (lengthNew <= 16ULL && lengthNew > 8ULL) { + return 16ULL; + } else { + return lengthNew; + } + }.template operator()(); + + template::length > 8)> struct pack_values; + + template struct pack_values { + static consteval decltype(auto) gen() noexcept { + using int_type = convert_length_to_int_t; + int_type returnValues{}; + for (uint64_t x = 0; x < string.length; ++x) { + if constexpr (std::endian::native == std::endian::little) { + returnValues |= static_cast(static_cast(static_cast(string[x])) << ((x % 8) * 8)); + } else { + returnValues |= static_cast(static_cast(static_cast(string[x])) << (((sizeof(int_type) - 1 - (x % 8))) * 8)); + } + } + return returnValues; + } + + static constexpr auto value{ gen() }; + }; + + template struct pack_values { + using array_type = array>; + + static consteval array_type gen() noexcept { + array_type outLocal{}; + for (uint64_t i = 0; i < string.size(); ++i) { + outLocal[i] = static_cast(string[i]); + } + return outLocal; + } + + alignas(simdBytesPerRegister) static constexpr auto value{ gen() }; + }; + + struct string_status { + uint64_t remainingLength{}; + uint64_t currentIndex{}; + }; + + template consteval string_literal<((offset >= string.length) ? string.length : offset) + 1> offSetIntoLiteral() { + constexpr uint64_t newSize = ((offset >= string.length) ? string.length : offset); + string_literal sl{}; + if constexpr (newSize > 0) { + std::copy_n(string.data(), newSize, sl.values); + sl.values[newSize] = '\0'; + } + return sl; + } + + template consteval string_literal<((offset >= string.length) ? 0 : string.length - offset) + 1> offSetNewLiteral() { + constexpr uint64_t newSize = (offset >= string.length) ? 0 : string.length - offset; + string_literal sl{}; + if constexpr (newSize > 0) { + std::copy_n(string.data() + offset, newSize, sl.values); + sl.values[newSize] = '\0'; + } + return sl; + } + + template consteval uint64_t getSplitStringCount(const string_literal&) noexcept { + constexpr uint64_t splitCount = (size + splitMod - 1ull) / splitMod; + return splitCount == 0ull ? 1ull : splitCount; + } + + template struct string_literal_splitter_impl { + static constexpr uint64_t currentLength{ splitMod < status.remainingLength ? splitMod : status.remainingLength }; + static constexpr auto string{ offSetIntoLiteral(), currentLength>() }; + }; + + template struct string_literal_splitter; + + consteval auto getOffsetIntoLiteralSize(uint64_t inputSize) noexcept { + if (inputSize >= 64ULL && simdBytesPerRegister >= 64ULL) { + return 64ULL; + } else if (inputSize >= 32ULL && simdBytesPerRegister >= 32ULL) { + return 32ULL; + } else { + return 16ULL; + } + } + + template stringNew, typename = void> struct string_literal_comparator_impl; + + template struct string_literal_splitter, stringNew, splitMod> { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + return (static_cast( + ((str = string_literal_comparator_impl::string), + string_literal_splitter_impl::string>::impl(str), + str != nullptr) && + ...)), + str); + } + + static consteval string_status getSplitStringStatus(uint64_t index, uint64_t totalLength) noexcept { + uint64_t currentIndex{ splitMod * index }; + string_status status{ totalLength, currentIndex }; + status.remainingLength -= currentIndex; + return status; + } + }; + + template stringNew> struct string_literal_comparator_impl { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + return str; + } + }; + + template stringNew> struct string_literal_comparator_impl { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + static constexpr auto stringLiteral{ stringNew }; + static constexpr auto newCount{ stringLiteral.size() }; + return (*str == stringLiteral[0]) ? str + newCount : nullptr; + } + }; + + template stringNew> struct string_literal_comparator_impl { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + static constexpr auto stringLiteral{ stringNew }; + static constexpr auto newCount{ stringLiteral.size() }; + static constexpr uint16_t valuesNew{ pack_values::value }; + uint16_t l; + pow2MemcpyWrapper<2>(&l, str); + return !(l ^ valuesNew) ? str + newCount : nullptr; + } + }; + + template stringNew> struct string_literal_comparator_impl { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + static constexpr auto stringLiteral{ stringNew }; + static constexpr auto newCount{ stringLiteral.size() }; + static constexpr auto loString = stringLiteral.template substr<0, 2>(); + static constexpr auto hiString = stringLiteral.template substr<1, 2>(); + static constexpr uint16_t loValuesNew{ pack_values::value }; + static constexpr uint16_t hiValuesNew{ pack_values::value }; + uint16_t lo, hi; + pow2MemcpyWrapper<2>(&lo, str); + pow2MemcpyWrapper<2>(&hi, str + 1); + return !((lo ^ loValuesNew) | (hi ^ hiValuesNew)) ? str + newCount : nullptr; + } + }; + + template stringNew> struct string_literal_comparator_impl { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + static constexpr auto stringLiteral{ stringNew }; + static constexpr auto newCount{ stringLiteral.size() }; + static constexpr uint32_t valuesNew{ pack_values::value }; + uint32_t l; + pow2MemcpyWrapper<4>(&l, str); + return !(l ^ valuesNew) ? str + newCount : nullptr; + } + }; + + template stringNew> struct string_literal_comparator_impl { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + static constexpr auto stringLiteral{ stringNew }; + static constexpr auto newCount{ stringLiteral.size() }; + static constexpr auto loString = stringLiteral.template substr<0, 4>(); + static constexpr auto hiString = stringLiteral.template substr<1, 4>(); + static constexpr uint32_t loValuesNew{ pack_values::value }; + static constexpr uint32_t hiValuesNew{ pack_values::value }; + uint32_t lo, hi; + pow2MemcpyWrapper<4>(&lo, str); + pow2MemcpyWrapper<4>(&hi, str + 1); + return !((lo ^ loValuesNew) | (hi ^ hiValuesNew)) ? str + newCount : nullptr; + } + }; + + template stringNew> struct string_literal_comparator_impl { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + static constexpr auto stringLiteral{ stringNew }; + static constexpr auto newCount{ stringLiteral.size() }; + static constexpr auto loString = stringLiteral.template substr<0, 4>(); + static constexpr auto hiString = stringLiteral.template substr<2, 4>(); + static constexpr uint32_t loValuesNew{ pack_values::value }; + static constexpr uint32_t hiValuesNew{ pack_values::value }; + uint32_t lo, hi; + pow2MemcpyWrapper<4>(&lo, str); + pow2MemcpyWrapper<4>(&hi, str + 2); + return !((lo ^ loValuesNew) | (hi ^ hiValuesNew)) ? str + newCount : nullptr; + } + }; + + template stringNew> struct string_literal_comparator_impl { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + static constexpr auto stringLiteral{ stringNew }; + static constexpr auto newCount{ stringLiteral.size() }; + static constexpr auto loString = stringLiteral.template substr<0, 4>(); + static constexpr auto hiString = stringLiteral.template substr<3, 4>(); + static constexpr uint32_t loValuesNew{ pack_values::value }; + static constexpr uint32_t hiValuesNew{ pack_values::value }; + uint32_t lo, hi; + pow2MemcpyWrapper<4>(&lo, str); + pow2MemcpyWrapper<4>(&hi, str + 3); + return !((lo ^ loValuesNew) | (hi ^ hiValuesNew)) ? str + newCount : nullptr; + } + }; + + template stringNew> struct string_literal_comparator_impl { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + static constexpr auto stringLiteral{ stringNew }; + static constexpr auto newCount{ stringLiteral.size() }; + static constexpr uint64_t valuesNew{ pack_values::value }; + uint64_t l; + pow2MemcpyWrapper<8>(&l, str); + return !(l ^ valuesNew) ? str + newCount : nullptr; + } + }; + + template stringNew> struct string_literal_comparator_impl { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + static constexpr auto stringLiteral{ stringNew }; + static constexpr auto newCount{ stringLiteral.size() }; + static constexpr auto loString = stringLiteral.template substr<0, 8>(); + static constexpr uint64_t loValuesNew{ pack_values::value }; +#if JSONIFIER_ARCH_X64 + static constexpr uint64_t tailBytes{ std::bit_ceil(newCount - 8) }; + using tail_type = convert_length_to_int_t; + static constexpr auto tailString = stringLiteral.template substr(); + static constexpr tail_type tailValuesNew{ pack_values::value }; + uint64_t lo; + tail_type tail; + pow2MemcpyWrapper<8>(&lo, str); + pow2MemcpyWrapper(&tail, str + newCount - tailBytes); + return !((lo ^ loValuesNew) | static_cast(static_cast(tail ^ tailValuesNew))) ? str + newCount : nullptr; +#else + static constexpr auto hiString = stringLiteral.template substr(); + static constexpr uint64_t hiValuesNew{ pack_values::value }; + uint64_t lo, hi; + pow2MemcpyWrapper<8>(&lo, str); + pow2MemcpyWrapper<8>(&hi, str + newCount - 8); + return !((lo ^ loValuesNew) | (hi ^ hiValuesNew)) ? str + newCount : nullptr; +#endif + } + }; + + template stringNew> struct string_literal_comparator_impl { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + static constexpr auto newLiteral{ stringNew }; + alignas(simdBytesPerRegister) static constexpr auto valuesNew{ pack_values::value }; + const jsonifier_simd_int_128 data1{ simd::gatherValuesU(str) }; + const jsonifier_simd_int_128 data2{ simd::gatherValues(valuesNew.data()) }; + return simd::opTest(simd::opXor(data1, data2)) ? str + 16 : nullptr; + } + }; + + template stringNew> struct string_literal_comparator_impl { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + static constexpr auto stringLiteral{ stringNew }; + static constexpr auto newCount{ stringLiteral.size() }; + static constexpr auto loString = stringLiteral.template substr<0, 16>(); + static constexpr auto hiString = stringLiteral.template substr(); + alignas(simdBytesPerRegister) static constexpr auto loValuesNew{ pack_values::value }; + alignas(simdBytesPerRegister) static constexpr auto hiValuesNew{ pack_values::value }; + const jsonifier_simd_int_128 lo{ simd::opXor(simd::gatherValuesU(str), simd::gatherValues(loValuesNew.data())) }; + const jsonifier_simd_int_128 hi{ simd::opXor(simd::gatherValuesU(str + newCount - 16), + simd::gatherValues(hiValuesNew.data())) }; + return simd::opTest(simd::opOr(lo, hi)) ? str + newCount : nullptr; + } + }; + +#if JSONIFIER_AVX512 || JSONIFIER_AVX2 + + template stringNew> struct string_literal_comparator_impl { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + static constexpr auto newLiteral{ stringNew }; + alignas(simdBytesPerRegister) static constexpr auto valuesNew{ pack_values::value }; + const jsonifier_simd_int_256 data1{ simd::gatherValuesU(str) }; + const jsonifier_simd_int_256 data2{ simd::gatherValues(valuesNew.data()) }; + return simd::opTest(simd::opXor(data1, data2)) ? str + 32 : nullptr; + } + }; + + template stringNew> struct string_literal_comparator_impl { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + static constexpr auto stringLiteral{ stringNew }; + static constexpr auto newCount{ stringLiteral.size() }; + static constexpr auto loString = stringLiteral.template substr<0, 32>(); + static constexpr auto hiString = stringLiteral.template substr(); + alignas(simdBytesPerRegister) static constexpr auto loValuesNew{ pack_values::value }; + alignas(simdBytesPerRegister) static constexpr auto hiValuesNew{ pack_values::value }; + const jsonifier_simd_int_256 lo{ simd::opXor(simd::gatherValuesU(str), simd::gatherValues(loValuesNew.data())) }; + const jsonifier_simd_int_256 hi{ simd::opXor(simd::gatherValuesU(str + newCount - 32), + simd::gatherValues(hiValuesNew.data())) }; + return simd::opTest(simd::opOr(lo, hi)) ? str + newCount : nullptr; + } + }; + +#endif + +#if JSONIFIER_AVX512 + template stringNew> struct string_literal_comparator_impl { + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + static constexpr auto newLiteral{ stringNew }; + alignas(simdBytesPerRegister) static constexpr auto valuesNew{ pack_values::value }; + const jsonifier_simd_int_512 data1{ simd::gatherValuesU(str) }; + const jsonifier_simd_int_512 data2{ simd::gatherValues(valuesNew.data()) }; + return simd::opCmpNeq(data1, data2) == 0 ? str + 64 : nullptr; + } + }; +#endif + + template stringNew> struct string_literal_comparator_impl { + static constexpr uint64_t splitMod{ getOffsetIntoLiteralSize(stringNew.size()) }; + static constexpr auto stringCount{ getSplitStringCount(stringNew) }; + + JSONIFIER_INLINE static unrestrict_t impl(read_buffer_ptr str) noexcept { + return string_literal_splitter, stringNew, splitMod>::impl(str); + } + }; + + template struct string_literal_comparator { + JSONIFIER_INLINE static bool impl(read_buffer_ptr str, uint64_t stringLength) noexcept { + using sl_type = decltype(string); + return stringLength >= string.size() ? string_literal_comparator_impl::impl(str) != nullptr : false; + } + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/concepts.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/concepts.hpp new file mode 100644 index 000000000000..1ca5f3bb095c --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/concepts.hpp @@ -0,0 +1,442 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/concepts.hpp + */ +#pragma once + +#include + +namespace jsonifier { + + enum class json_types : uint8_t { + object = 0, + array = 1, + string = 2, + number = 3, + boolean = 4, + null = 5, + unset = 8, + count, + }; + + class string_view_base; + + template class string_base; + + class raw_json_data; + + // Idea for this interface sampled from Stephen Berry and his library, Glaze library: https://github.com/stephenberry/glaze + template struct core; + + struct skip {}; + +} + +namespace jsonifier::internal { + + template + concept structural_context_types = std::same_as>; + + template using context_iterator_t = typename context_type::iterator_type; + + template + concept simd_int_512_types = sizeof(value_type) == 64; + template + concept simd_int_256_types = sizeof(value_type) == 32; + template + concept simd_int_128_types = sizeof(value_type) == 16; + +#if JSONIFIER_SVE2 + template + concept simd_int_sve2_types = std::same_as, jsonifier_simd_int_t>; +#endif + + template + concept json_entity_types = requires { + typename value_type::class_type; + value_type::memberPtr; + }; + + template + concept json_entity_convertible_types = json_entity_types || std::is_member_pointer_v; + + template using base_t = remove_cvref_t; + + template inline constexpr bool isSame = false; + template inline constexpr bool isSame = true; + + template + concept static_sized_types = requires() { base_t::staticSize; }; + + template + concept same_as_types = isSame, remove_reference_t>; + + template + concept enum_types = std::is_enum_v> && std::is_unsigned_v>>; + + template + concept integral_types = std::is_integral_v> && !same_as_types, bool>; + + template + concept trivially_copyable_types = __is_trivial(base_t) || std::is_standard_layout_v>; + + template + concept trivially_destructible_types = std::is_trivially_destructible_v>; + + template + concept trivially_constructible_types = std::is_trivially_constructible_v>; + + template + concept trivially_constructible_copyable_destructible_types = + trivially_constructible_types && trivially_copyable_types && trivially_destructible_types; + + template + concept int_types = std::is_signed_v> && integral_types; + + template + concept int8_types = int_types && sizeof(remove_cvref_t) == 1; + + template + concept int16_types = int_types && sizeof(remove_cvref_t) == 2; + + template + concept int32_types = int_types && sizeof(remove_cvref_t) == 4; + + template + concept int64_types = int_types && sizeof(remove_cvref_t) == 8; + + template + concept uint_types = std::is_unsigned_v> && integral_types; + + template + concept uint8_types = uint_types && sizeof(remove_cvref_t) == 1; + + template + concept uint16_types = uint_types && sizeof(remove_cvref_t) == 2; + + template + concept uint32_types = uint_types && sizeof(remove_cvref_t) == 4; + + template + concept uint64_types = uint_types && sizeof(remove_cvref_t) == 8; + + template + concept float_types = std::floating_point> && (std::numeric_limits>::radix == 2) && std::numeric_limits>::is_iec559; + + template + concept float64_types = float_types && sizeof(remove_cvref_t) == 8; + + template + concept float32_types = float_types && sizeof(remove_cvref_t) == 4; + + template + concept string_view_convertible_types = std::convertible_to, std::string_view>; + + template + concept at_least_one_enum_types = ( enum_types && uint_types ) || (uint_types && enum_types); + + template + concept indexable_types = + at_least_one_enum_types || same_as_types || (integral_types && integral_types); + + template + concept resize_and_overwriteable_types = requires(value_type value) { + value.resize_and_overwrite(uint64_t{}, [](typename value_type::pointer, uint64_t) noexcept { + return uint64_t{}; + }); + }; + + template + concept skip_types = std::is_same_v, skip>; + + template + concept iterable_types = requires(base_t value) { + { value.begin() }; + { value.end() }; + }; + + template + concept map_subscriptable_types = requires(base_t value) { + { value[std::declval::key_type>()] } -> std::same_as::mapped_type&>; + } || requires(base_t value) { + { value[std::declval::key_type>()] } -> std::same_as::mapped_type&>; + }; + + template + concept vector_subscriptable_types = requires(base_t value) { + { value[std::declval::size_type>()] } -> std::same_as::const_reference>; + } || requires(base_t value) { + { value[std::declval::size_type>()] } -> std::same_as::reference>; + }; + + template + concept sized_types = requires(base_t value) { + { value.size() } -> std::same_as::size_type>; + }; + + template + concept empty_checkable_types = requires(base_t value) { + { value.empty() } -> std::same_as; + }; + + template + concept variant_types = requires(base_t var) { + { var.index() } -> std::convertible_to; + { var.valueless_by_exception() } -> std::same_as; + { std::holds_alternative(var))>(var) } -> std::same_as; + { std::get<0>(var) } -> std::same_as(var))&>; + { std::get_if<0>(&var) } -> std::same_as(var))>*>; + }; + + template + concept resizeable_types = requires(base_t value) { value.resize(std::declval::size_type>()); }; + + template + concept reserveable_types = requires(base_t value) { value.reserve(std::declval::size_type>()); }; + + template + concept data_accessible_types = requires(base_t value) { + { value.data() }; + }; + + template + concept bool_types = std::same_as, bool> || std::same_as, std::vector::reference> || + std::same_as, std::vector::const_reference>; + + template + concept always_null_types = + std::same_as, std::nullptr_t> || std::same_as, std::monostate> || std::same_as, std::nullopt_t>; + + template + concept pointer_types = + (std::is_pointer_v> || ( std::is_null_pointer_v> && !std::is_array_v> )) && !always_null_types; + + template + concept char_types = std::same_as, char> || std::same_as, wchar_t> || std::same_as, char8_t> || + std::same_as, char16_t> || std::same_as, char32_t>; + + template + concept number_types = (float_types || uint_types || int_types) && !char_types; + + template + concept substringable_types = requires(base_t value) { + { value.substr(std::declval::size_type>(), std::declval::size_type>()) } -> std::same_as>; + }; + + template + concept findable_types = requires(base_t value) { + { value.find(std::declval::value_type>()) } -> std::same_as::size_type>; + } || requires(base_t value) { + { value.find(std::declval::key_type>()) } -> std::same_as::iterator>; + } || requires(base_t value) { + { value.find(std::declval::key_type>()) } -> std::same_as::const_iterator>; + }; + + template + concept string_types = + substringable_types && data_accessible_types && sized_types && vector_subscriptable_types && findable_types; + + template + concept string_view_types = substringable_types && data_accessible_types && sized_types && vector_subscriptable_types && + findable_types && !resizeable_types; + + template + concept map_types = + map_subscriptable_types && iterable_types && sized_types && findable_types && empty_checkable_types; + + template + concept emplace_backable_types = requires(base_t value) { + { value.emplace_back(std::declval::value_type>()) } -> std::same_as::reference>; + }; + + template + concept releasable_types = requires(base_t value) { + { value.release() } -> std::same_as::pointer>; + }; + + template + concept resettable_types = requires(base_t value) { + { value.reset() } -> std::same_as; + }; + + template + concept gettable_types = requires(base_t value) { + { value.get() } -> std::same_as::element_type*>; + }; + + template + concept copyable_types = std::copyable>; + + template + concept unique_ptr_types = requires(base_t value) { + typename base_t::element_type; + typename base_t::deleter_type; + } && releasable_types && gettable_types; + + template + concept shared_ptr_types = resettable_types && gettable_types && copyable_types; + + template + concept any_pointer_types = unique_ptr_types || shared_ptr_types || pointer_types; + + template + concept excluded_keys_types = requires(base_t value) { + { value.jsonifierExcludedKeys }; + }; + + template + concept raw_json_types = std::same_as, raw_json_data>; + + template + concept same_character_size_types = requires { sizeof(typename base_t::value_type) == sizeof(typename base_t::value_type); } && + string_types && string_types; + + template inline constexpr bool hasSizeEqualToZero{ std::tuple_size_v> == 0 }; + + template + concept skip_or_always_null_types = skip_types || always_null_types; + + template + concept tuple_types = requires { std::tuple_size>::value; } && !data_accessible_types; + + template using decay_keep_volatile_t = std::remove_const_t>; + + template + concept optional_types = requires(base_t opt) { + { opt.has_value() } -> std::same_as; + { opt.value() } -> std::same_as::value_type&>; + { *opt } -> std::same_as::value_type&>; + { opt.reset() } -> std::same_as; + { opt.emplace(std::declval::value_type>()) } -> std::same_as::value_type&>; + }; + + template + concept any_pointer_or_optional_types = any_pointer_types || optional_types; + + template + concept any_enum_types = std::is_enum_v>; + + template + concept vector_types = vector_subscriptable_types && resizeable_types && emplace_backable_types; + + template + concept float_vector_types = vector_types && float_types::value_type>; + + template + concept jsonifier_object_types = requires { core>::parseValue; }; + + template + concept raw_array_types = ( std::is_array_v> && !std::is_pointer_v> ) || + (vector_subscriptable_types && !vector_types && !substringable_types && !tuple_types); + + template + concept buffer_like_types = vector_subscriptable_types && data_accessible_types && resizeable_types; + + template + concept integer_types = std::integral> && !bool_types && !std::floating_point>; + + template + concept count_types = requires { base_t::count; } && static_cast(base_t::count) < 128; + + template + concept printable_enum_types = enum_types && count_types; + + template + concept integral_constant_types = requires { + { base_t::value } -> std::convertible_to; + }; + + template + concept eq_0_types = base_t::length == 0; + + template + concept gt_0_lte_8_types = base_t::length > 0 && base_t::length <= 8; + + template + concept gt_8_types = base_t::length > 8; + + template + concept eq_1_types = base_t::length == 1; + + template + concept eq_2_types = base_t::length == 2; + + template + concept eq_3_types = base_t::length == 3; + + template + concept eq_4_types = base_t::length == 4; + + template + concept eq_5_types = base_t::length == 5; + + template + concept eq_6_types = base_t::length == 6; + + template + concept eq_7_types = base_t::length == 7; + + template + concept eq_8_types = base_t::length == 8; + + template + concept gt_8_lt_16_types = base_t::length > 8 && base_t::length < 16; + + template + concept eq_16_types = base_t::length == 16 && simdBytesPerRegister >= 16; + + template + concept eq_32_types = base_t::length == 32 && simdBytesPerRegister >= 32; + + template + concept eq_64_types = base_t::length == 64 && simdBytesPerRegister >= 64; + + template + concept gt_16_types = base_t::length > 16 && !eq_16_types && !eq_32_types && !eq_64_types; + + template + concept gt_16_lt_32_types = gt_16_types && base_t::length < 32; + + template + concept gt_32_lt_64_types = gt_16_types && base_t::length > 32 && base_t::length < 64; + + template + concept named_types = requires(base_t value) { value.name; }; + + template + concept derivable_types = std::is_class_v && !std::is_final_v; + + template + concept eq_comparable_types = requires { std::declval() == std::declval(); }; + + template + concept neq_comparable_types = requires { std::declval() != std::declval(); }; + + template + concept lt_comparable_types = requires { std::declval() < std::declval(); }; + + template + concept lte_comparable_types = requires { std::declval() <= std::declval(); }; + + template + concept gt_comparable_types = requires { std::declval() > std::declval(); }; + + template + concept gte_comparable_types = requires { std::declval() >= std::declval(); }; + + template + concept ss_comparable_types = requires { std::declval() <=> std::declval(); }; + + template typename template_type> struct is_specialization_of_impl : std::false_type {}; + + template typename template_type, typename... args> struct is_specialization_of_impl, template_type> : std::true_type {}; + + template typename template_type> + concept specialization_of_types = is_specialization_of_impl::value; + + template + concept prettify_buffer_types = string_types || std::is_convertible_v; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/error.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/error.hpp new file mode 100644 index 000000000000..0a88ab3deb61 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/error.hpp @@ -0,0 +1,309 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/error.hpp + */ +#pragma once + +#include +#include +#include +#include + +namespace jsonifier::internal { + + enum class status_classes : uint8_t { + unset, + parsing, + minifying, + prettifying, + validating, + count, + }; + + enum class parse_statuses : uint8_t { + success, + missing_key_start, + missing_object_start, + missing_object_end, + missing_array_start, + missing_array_end, + invalid_string_characters, + missing_colon, + missing_comma, + invalid_number_value, + invalid_null_value, + invalid_bool_value, + no_input, + unfinished_input, + unexpected_string_end, + unexpected_end_of_input, + exceeded_max_depth, + unexpected_token, + illegal_control_character, + count, + }; + + enum class minify_statuses : uint8_t { + success, + no_input, + invalid_string_length, + invalid_number_value, + incorrect_structural_index, + unclosed_object, + unclosed_array, + count, + }; + + enum class prettify_statuses : uint8_t { + success, + no_input, + exceeded_max_depth, + incorrect_structural_index, + unclosed_object, + unclosed_array, + count, + }; + + enum class validate_statuses : uint8_t { + success, + missing_object_start, + missing_object_end, + missing_array_start, + missing_array_end, + invalid_string_characters, + missing_colon, + missing_comma, + invalid_number_value, + invalid_null_value, + invalid_bool_value, + invalid_escape_characters, + missing_comma_or_closing_brace, + no_input, + count, + }; + + inline void appendErrorType(status_classes status_class, string& stream, uint64_t value) noexcept { + switch (static_cast(status_class)) { + case static_cast(status_classes::parsing): { + stream += getName(static_cast(value)); + return; + } + case static_cast(status_classes::minifying): { + stream += getName(static_cast(value)); + return; + } + case static_cast(status_classes::prettifying): { + stream += getName(static_cast(value)); + return; + } + case static_cast(status_classes::validating): { + stream += getName(static_cast(value)); + return; + } + default: { + return; + } + } + } + + inline void appendUnsigned(string& stream, uint64_t value) noexcept { + uint8_t buffer[24]; + auto* end = to_chars::impl(buffer, value); + stream.append(std::bit_cast(+buffer), static_cast(end - buffer)); + } + + inline void appendEscapedByte(string& stream, char b) noexcept { + switch (b) { + case '\a': + stream += "\\a"; + return; + case '\b': + stream += "\\b"; + return; + case '\f': + stream += "\\f"; + return; + case '\n': + stream += "\\n"; + return; + case '\r': + stream += "\\r"; + return; + case '\t': + stream += "\\t"; + return; + case '\v': + stream += "\\v"; + return; + case '\\': + stream += "\\\\"; + return; + case '\'': + stream += "\\'"; + return; + case '\"': + stream += "\\\""; + return; + case '\0': + stream += "\\0"; + return; + default: + if (std::isprint(static_cast(b))) { + stream += b; + return; + } else { + static constexpr char hexDigits[]{ "0123456789abcdef" }; + stream += "\\x"; + stream += hexDigits[(static_cast(b) >> 4U) & 0xFU]; + stream += hexDigits[static_cast(b) & 0xFU]; + return; + } + } + } + + enum class json_structural_types : int8_t { + error = -1, + unset = 0, + string = '"', + comma = ',', + number = '-', + colon = ':', + array_start = '[', + array_end = ']', + null = 'n', + boolean = 't', + object_start = '{', + object_end = '}', + }; + + consteval array genBoolTable() noexcept { + array returnValues{}; + returnValues[uint_c_v<'t'>] = true; + returnValues[uint_c_v<'f'>] = true; + return returnValues; + } + + alignas(64) inline constexpr auto boolTableRaw{ genBoolTable() }; + + inline constexpr const bool* __restrict boolTable{ boolTableRaw.data() }; + + struct error { + inline error() noexcept = default; + + template error(std::source_location sourceLocationNew, status_classes statusClassNew, read_buffer_ptr rootIterNew, read_buffer_ptr errorPosNew, + read_buffer_ptr endIterNew, error_class typeNew) noexcept + : sourceLocation{ sourceLocationNew }, rootIter{ rootIterNew }, errorPos{ errorPosNew }, endIter{ endIterNew }, errorType{ static_cast(typeNew) }, + status_class{ statusClassNew } { + formatReport(); + } + + template inline static error constructError(error_class typeNew, read_buffer_ptr rootIter, read_buffer_ptr errorPos, + read_buffer_ptr endIter, const std::source_location& sourceLocation = std::source_location::current()) noexcept { + return { sourceLocation, status_class, rootIter, errorPos, endIter, typeNew }; + } + + template inline static error constructError(read_buffer_ptr rootIter, read_buffer_ptr errorPos, read_buffer_ptr endIter, + const std::source_location& sourceLocation = std::source_location::current()) noexcept { + return { sourceLocation, status_class, rootIter, errorPos, endIter, typeNew }; + } + + inline bool operator==(const error& rhs) const noexcept { + return status_class == rhs.status_class && errorType == rhs.errorType && errorIndex == rhs.errorIndex && line == rhs.line && localIndex == rhs.localIndex; + } + + inline void formatReport() noexcept { + line = 0; + localIndex = 0; + errorIndex = 0; + string context{}; + if (errorPos && endIter && rootIter && errorPos >= rootIter && errorPos <= endIter) { + errorIndex = static_cast(errorPos - rootIter); + + line = static_cast(std::count(rootIter, errorPos, '\n') + 1); + + read_buffer_ptr scan = errorPos; + uint64_t distanceFromLineStart = 0; + while (scan > rootIter && *(scan - 1) != '\n') { + --scan; + ++distanceFromLineStart; + } + localIndex = distanceFromLineStart; + + if (errorPos < endIter) { + uint64_t errorLength = std::min(static_cast(16ULL), static_cast(endIter - errorPos)); + string_view view{ std::bit_cast(errorPos), errorLength }; + context = string{ std::begin(view), static_cast(view.size()) }; + for (auto& c: context) { + if (c == '\t') { + c = ' '; + } + } + } + } + string stream{}; + stream += "Error of Class: "; + stream += getName(status_class); + stream += ", of Type: "; + appendErrorType(status_class, stream, errorType); + stream += ", at global index: "; + appendUnsigned(stream, errorIndex); + stream += ", on line: "; + appendUnsigned(stream, line); + stream += ", at local index: "; + appendUnsigned(stream, localIndex); + if (!context.empty()) { + stream += "\nHere's some of the string's values: "; + collectValues(stream, context); + } + stream += "\nIn file: "; + stream += sourceLocation.file_name(); + stream += ", at line/column: "; + appendUnsigned(stream, static_cast(sourceLocation.line())); + stream += ":"; + appendUnsigned(stream, static_cast(sourceLocation.column())); + stream += "\n"; + reportString = stream; + } + + inline static void collectValues(string& stream, const string& inputValues) noexcept { + for (uint64_t i = 0; i < 32 && i < inputValues.size(); ++i) { + stream += "'"; + appendEscapedByte(stream, inputValues[i]); + stream += "' "; + } + return; + } + + template inline operator error_class() const noexcept { + return static_cast(errorType); + } + + inline std::string_view reportError() const noexcept { + return reportString.operator std::basic_string_view(); + } + + std::source_location sourceLocation{}; + read_buffer_ptr rootIter{}; + read_buffer_ptr errorPos{}; + read_buffer_ptr endIter{}; + string reportString{}; + uint64_t localIndex{}; + uint64_t errorIndex{}; + uint64_t errorType{}; + uint64_t line{}; + status_classes status_class{}; + }; + + template inline basic_stream& operator<<(basic_stream& os, const error& errorNew) noexcept { + os << errorNew.reportError(); + return os; + } + + inline std::ostream& operator<<(std::ostream& os, const error& errorNew) noexcept { + os << errorNew.reportError(); + return os; + } + +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/fast_float.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/fast_float.hpp new file mode 100644 index 000000000000..ddd8129bfa7e --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/fast_float.hpp @@ -0,0 +1,1439 @@ +/* + * SPDX-License-Identifier: MIT AND (Apache-2.0 OR MIT OR BSL-1.0) + * Copyright (c) 2026 Nihilai Collective Corp + * Portions Copyright (c) 2021 The fast_float authors (modified for jsonifier) + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/fast_float.hpp + */ +#pragma once + +// jsonifier_fast_float by Daniel Lemire +// jsonifier_fast_float by João Paulo Magalhaes +// +// +// with contributions from Eugene Golushkov +// with contributions from Maksim Kita +// with contributions from Marcin Wojdyr +// with contributions from Neal Richardson +// with contributions from Tim Paine +// with contributions from Fabio Pellacini +// with contributions from Lénárd Szolnoki +// with contributions from Jan Pharago +// with contributions from Maya Warrier +// with contributions from Taha Khokhar +// +// +// Licensed under the Apache License, Version 2.0, or the +// MIT License or the Boost License. This file may not be copied, +// modified, or distributed except according to those terms. +// +// MIT License Notice +// +// MIT License +// +// Copyright (c) 2021 The jsonifier_fast_float authors +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// +// Apache License (Version 2.0) Notice +// +// Copyright 2021 The jsonifier_fast_float authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// +// BOOST License Notice +// +// Boost Software License - Version 1.0 - August 17th, 2003 +// +// Permission is hereby granted, free of charge, to any person or organization +// obtaining a copy of the software and accompanying documentation covered by +// this license (the "Software") to use, reproduce, display, distribute, +// execute, and transmit the Software, and to prepare derivative works of the +// Software, and to permit third-parties to whom the Software is furnished to +// do so, all subject to the following: +// +// The copyright notices in the Software and this entire statement, including +// the above license grant, this restriction and the following disclaimer, +// must be included in all copies of the Software, in whole or in part, and +// all derivative works of the Software, unless such copies or derivative +// works are solely in the form of machine-executable object code generated by +// a source language processor. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +#include +#include +#include + +namespace jsonifier::internal { + + template JSONIFIER_INLINE bool isDigit(v_type value) noexcept { + return ((static_cast(value - '0')) < 10); + } + +#ifndef FLT_EVAL_METHOD + #error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif + + template struct span { + JSONIFIER_INLINE constexpr span() noexcept : ptr{ nullptr }, end{ nullptr } { + } + + JSONIFIER_INLINE constexpr span(const value_type* _ptr, const value_type* _end) noexcept : ptr{ _ptr }, end{ _end } { + } + + JSONIFIER_INLINE constexpr span(const value_type* _ptr) noexcept : ptr{ _ptr }, end{ nullptr } { + } + + const value_type* ptr; + const value_type* end; + }; + + struct value_128 { + JSONIFIER_INLINE constexpr value_128() noexcept : high{ 0 }, low{ 0 } { + } + + JSONIFIER_INLINE constexpr value_128(uint64_t _low, uint64_t _high) noexcept : high{ _high }, low{ _low } { + } + + uint64_t high; + uint64_t low; + }; + + JSONIFIER_INLINE constexpr uint64_t emulu(uint32_t x, uint32_t y) noexcept { + return x * static_cast(y); + } + + JSONIFIER_INLINE constexpr uint64_t umul128Generic(uint64_t ab, uint64_t cd, uint64_t* hi) noexcept { + uint64_t ad = emulu(static_cast(ab >> 32), static_cast(cd)); + uint64_t bd = emulu(static_cast(ab), static_cast(cd)); + uint64_t adbc = ad + emulu(static_cast(ab), static_cast(cd >> 32)); + uint64_t adbcCarry = static_cast(adbc < ad); + uint64_t lo = bd + (adbc << 32); + *hi = emulu(static_cast(ab >> 32), static_cast(cd >> 32)) + (adbc >> 32) + (adbcCarry << 32) + static_cast(lo < bd); + return lo; + } + + JSONIFIER_INLINE value_128 fullMultiplication(uint64_t a, uint64_t b) noexcept { + value_128 answer; +#if JSONIFIER_PLATFORM_WINDOWS && JSONIFIER_ARCH_ARM64 + + answer.high = __umulh(a, b); + answer.low = a * b; +#elif (JSONIFIER_PLATFORM_WINDOWS && !JSONIFIER_COMPILER_CLANG && JSONIFIER_ARCH_X64) + answer.low = _umul128(a, b, &answer.high); +#elif defined(__SIZEOF_INT128__) + __uint128_t r = (static_cast<__uint128_t>(a)) * b; + answer.low = static_cast(r); + answer.high = static_cast(r >> 64); +#else + answer.low = umul128Generic(a, b, &answer.high); +#endif + return answer; + } + + struct adjusted_mantissa { + JSONIFIER_INLINE adjusted_mantissa() noexcept = default; + + JSONIFIER_INLINE constexpr bool operator==(const adjusted_mantissa& o) const noexcept { + return mantissa == o.mantissa && power2 == o.power2; + } + + JSONIFIER_INLINE constexpr bool operator!=(const adjusted_mantissa& o) const noexcept { + return mantissa != o.mantissa || power2 != o.power2; + } + + uint64_t mantissa; + int32_t power2; + }; + + inline constexpr int32_t invalidAmBias = -0x8000; + + inline constexpr uint64_t constant55555 = 5 * 5 * 5 * 5 * 5; + + template struct binary_format_lookup_tables; + + template struct binary_format : binary_format_lookup_tables { + using equiv_uint = typename std::conditional::type; + + static constexpr int32_t mantissaExplicitBits = (sizeof(value_type) == 4) ? 23 : 52; + static constexpr int32_t minimumExponent = (sizeof(value_type) == 4) ? -127 : -1023; + static constexpr int32_t infinitePower = (sizeof(value_type) == 4) ? 0xFF : 0x7FF; + static constexpr int32_t signIndex = (sizeof(value_type) == 4) ? 31 : 63; + static constexpr int32_t minExponentFastPath = (sizeof(value_type) == 4) ? -10 : -22; + static constexpr int32_t maxExponentFastPath = (sizeof(value_type) == 4) ? 10 : 22; + static constexpr int32_t maxExponentRoundToEven = (sizeof(value_type) == 4) ? 10 : 23; + static constexpr int32_t minExponentRoundToEven = (sizeof(value_type) == 4) ? -17 : -4; + static constexpr uint64_t maxMantissaFastPathValue = 2ull << mantissaExplicitBits; + static constexpr int32_t largestPowerOfTen = (sizeof(value_type) == 4) ? 38 : 308; + static constexpr int32_t smallestPowerOfTen = (sizeof(value_type) == 4) ? -64 : -342; + static constexpr uint64_t maxDigits = (sizeof(value_type) == 4) ? 114 : 769; + static constexpr equiv_uint exponentMask = (sizeof(value_type) == 4) ? 0x7F800000 : 0x7FF0000000000000; + static constexpr equiv_uint mantissaMask = (sizeof(value_type) == 4) ? 0x007FFFFF : 0x000FFFFFFFFFFFFF; + static constexpr equiv_uint hiddenBitMask = (sizeof(value_type) == 4) ? 0x00800000 : 0x0010000000000000; + + JSONIFIER_INLINE static constexpr uint64_t maxMantissaFastPath(int64_t power) noexcept { + return binary_format_lookup_tables::maxMantissa[power]; + } + + JSONIFIER_INLINE static constexpr value_type exactPowerOfTen(int64_t power) noexcept { + return binary_format_lookup_tables::powersOfTen[power]; + } + }; + + template struct binary_format_lookup_tables { + alignas(64) static constexpr double powersOfTen[]{ 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, + 1e22 }; + + alignas(64) static constexpr uint64_t maxMantissa[]{ 0x20000000000000, 0x20000000000000 / 5, 0x20000000000000 / (5 * 5), 0x20000000000000 / (5 * 5 * 5), + 0x20000000000000 / (5 * 5 * 5 * 5), 0x20000000000000 / (constant55555), 0x20000000000000 / (constant55555 * 5), 0x20000000000000 / (constant55555 * 5 * 5), + 0x20000000000000 / (constant55555 * 5 * 5 * 5), 0x20000000000000 / (constant55555 * 5 * 5 * 5 * 5), 0x20000000000000 / (constant55555 * constant55555), + 0x20000000000000 / (constant55555 * constant55555 * 5), 0x20000000000000 / (constant55555 * constant55555 * 5 * 5), + 0x20000000000000 / (constant55555 * constant55555 * 5 * 5 * 5), 0x20000000000000 / (constant55555 * constant55555 * constant55555), + 0x20000000000000 / (constant55555 * constant55555 * constant55555 * 5), 0x20000000000000 / (constant55555 * constant55555 * constant55555 * 5 * 5), + 0x20000000000000 / (constant55555 * constant55555 * constant55555 * 5 * 5 * 5), 0x20000000000000 / (constant55555 * constant55555 * constant55555 * 5 * 5 * 5 * 5), + 0x20000000000000 / (constant55555 * constant55555 * constant55555 * constant55555), + 0x20000000000000 / (constant55555 * constant55555 * constant55555 * constant55555 * 5), + 0x20000000000000 / (constant55555 * constant55555 * constant55555 * constant55555 * 5 * 5), + 0x20000000000000 / (constant55555 * constant55555 * constant55555 * constant55555 * 5 * 5 * 5), + 0x20000000000000 / (constant55555 * constant55555 * constant55555 * constant55555 * 5 * 5 * 5 * 5) }; + }; + + template struct binary_format_lookup_tables { + alignas(64) static constexpr float powersOfTen[]{ 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, 1e6f, 1e7f, 1e8f, 1e9f, 1e10f }; + + alignas(64) static constexpr uint64_t maxMantissa[]{ 0x1000000, 0x1000000 / 5, 0x1000000 / (5 * 5), 0x1000000 / (5 * 5 * 5), 0x1000000 / (5 * 5 * 5 * 5), + 0x1000000 / (constant55555), 0x1000000 / (constant55555 * 5), 0x1000000 / (constant55555 * 5 * 5), 0x1000000 / (constant55555 * 5 * 5 * 5), + 0x1000000 / (constant55555 * 5 * 5 * 5 * 5), 0x1000000 / (constant55555 * constant55555), 0x1000000 / (constant55555 * constant55555 * 5) }; + }; + + template JSONIFIER_INLINE constexpr void toFloat(bool negative, adjusted_mantissa am, value_type& value) noexcept { + using fastfloat_uint = typename binary_format::equiv_uint; + fastfloat_uint word = static_cast(am.mantissa); + word |= fastfloat_uint(am.power2) << binary_format::mantissaExplicitBits; + word |= fastfloat_uint(negative) << binary_format::signIndex; + value = std::bit_cast(word); + } + + template inline constexpr uint64_t intCmpZeros{ (sizeof(char_types) == 1) ? 0x3030303030303030 + : (sizeof(char_types) == 2) + ? (static_cast(char_types('0')) << 48 | static_cast(char_types('0')) << 32 | static_cast(char_types('0')) << 16 | char_types('0')) + : (static_cast(char_types('0')) << 32 | char_types('0')) }; + template inline constexpr int32_t intCmpLen{ sizeof(uint64_t) / sizeof(char_types) }; + + template struct int_luts { + alignas(64) static constexpr uint8_t chdigit[]{ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, + 255, 255, 255, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 255, 255, 255, 255, 255, 255, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 }; + + alignas(64) static constexpr uint64_t maxdigitsU64[]{ 64, 41, 32, 28, 25, 23, 22, 21, 20, 19, 18, 18, 17, 17, 16, 16, 16, 16, 15, 15, 15, 15, 14, 14, 14, 14, 14, 14, 14, + 13, 13, 13, 13, 13, 13 }; + + alignas(64) static constexpr uint64_t minSafeU64[]{ 9223372036854775808ull, 12157665459056928801ull, 4611686018427387904, 7450580596923828125, 4738381338321616896, + 3909821048582988049, 9223372036854775808ull, 12157665459056928801ull, 10000000000000000000ull, 5559917313492231481, 2218611106740436992, 8650415919381337933, + 2177953337809371136, 6568408355712890625, 1152921504606846976, 2862423051509815793, 6746640616477458432, 15181127029874798299ull, 1638400000000000000, + 3243919932521508681, 6221821273427820544, 11592836324538749809ull, 876488338465357824, 1490116119384765625, 2481152873203736576, 4052555153018976267, + 6502111422497947648, 10260628712958602189ull, 15943230000000000000ull, 787662783788549761, 1152921504606846976, 1667889514952984961, 2386420683693101056, + 3379220508056640625, 4738381338321616896 }; + }; + + alignas(64) inline constexpr uint64_t pow10Table[]{ 1ull, 10ull, 100ull, 1000ull, 10000ull, 100000ull, 1000000ull, 10000000ull, 100000000ull, 1000000000ull, 10000000000ull, + 100000000000ull }; + + template struct parse_x_digits_unrolled; + + template JSONIFIER_INLINE constexpr uint64_t read8ToU64(char_type const* chars) noexcept { + uint64_t val; + pow2MemcpyWrapper(&val, chars); + if constexpr (std::endian::native == std::endian::big) { + val = internal::byteswap(val); + } + return val; + } + + template JSONIFIER_INLINE constexpr uint32_t read4ToU32(char_type const* chars) noexcept { + uint32_t val; + pow2MemcpyWrapper(&val, chars); + if constexpr (std::endian::native == std::endian::big) { + val = internal::byteswap(val); + } + return val; + } + + JSONIFIER_INLINE constexpr bool isMadeOfEightDigitsFast(uint64_t val) noexcept { + return !((((val + 0x4646464646464646) | (val - 0x3030303030303030)) & 0x8080808080808080)); + } + + JSONIFIER_INLINE constexpr uint32_t parseFourDigitsUnrolled(uint32_t val) noexcept { + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; + } + + JSONIFIER_INLINE constexpr bool isMadeOfFourDigitsFast(uint32_t val) noexcept { + return !((((val + 0x46464646) | (val - 0x30303030)) & 0x80808080)); + } + + JSONIFIER_INLINE uint32_t parseEightDigitsUnrolled(uint64_t val) noexcept { + const uint64_t mask = 0x000000FF000000FF; + const uint64_t mul1 = 0x000F424000000064; + const uint64_t mul2 = 0x0000271000000001; + val -= 0x3030303030303030; + val = (val * 10) + (val >> 8); + val = (((val & mask) * mul1) + (((val >> 16) & mask) * mul2)) >> 32; + return static_cast(val); + } + + JSONIFIER_INLINE constexpr void loopParseIfEightDigits(uint8_t const* __restrict& p, uint8_t const* const pend, uint64_t& i) noexcept { + while (((pend - p) >= 8) && isMadeOfEightDigitsFast(read8ToU64(p))) { + i = i * 100000000 + parseEightDigitsUnrolled(read8ToU64(p)); + p += 8; + } + if ((pend - p) >= 4) { + uint32_t const val4 = read4ToU32(p); + if (isMadeOfFourDigitsFast(val4)) { + i = i * 10000 + parseFourDigitsUnrolled(val4); + p += 4; + } + } + } + + template struct powers_template { + static constexpr int32_t smallestPowerOfFive = binary_format::smallestPowerOfTen; + static constexpr int32_t largestPowerOfFive = binary_format::largestPowerOfTen; + static constexpr int32_t numberOfEntries = 2 * (largestPowerOfFive - smallestPowerOfFive + 1); + + alignas(64) static constexpr uint64_t powerOfFive128[numberOfEntries]{ 0xeef453d6923bd65a, 0x113faa2906a13b3f, 0x9558b4661b6565f8, 0x4ac7ca59a424c507, 0xbaaee17fa23ebf76, + 0x5d79bcf00d2df649, 0xe95a99df8ace6f53, 0xf4d82c2c107973dc, 0x91d8a02bb6c10594, 0x79071b9b8a4be869, 0xb64ec836a47146f9, 0x9748e2826cdee284, 0xe3e27a444d8d98b7, + 0xfd1b1b2308169b25, 0x8e6d8c6ab0787f72, 0xfe30f0f5e50e20f7, 0xb208ef855c969f4f, 0xbdbd2d335e51a935, 0xde8b2b66b3bc4723, 0xad2c788035e61382, 0x8b16fb203055ac76, + 0x4c3bcb5021afcc31, 0xaddcb9e83c6b1793, 0xdf4abe242a1bbf3d, 0xd953e8624b85dd78, 0xd71d6dad34a2af0d, 0x87d4713d6f33aa6b, 0x8672648c40e5ad68, 0xa9c98d8ccb009506, + 0x680efdaf511f18c2, 0xd43bf0effdc0ba48, 0x212bd1b2566def2, 0x84a57695fe98746d, 0x14bb630f7604b57, 0xa5ced43b7e3e9188, 0x419ea3bd35385e2d, 0xcf42894a5dce35ea, + 0x52064cac828675b9, 0x818995ce7aa0e1b2, 0x7343efebd1940993, 0xa1ebfb4219491a1f, 0x1014ebe6c5f90bf8, 0xca66fa129f9b60a6, 0xd41a26e077774ef6, 0xfd00b897478238d0, + 0x8920b098955522b4, 0x9e20735e8cb16382, 0x55b46e5f5d5535b0, 0xc5a890362fddbc62, 0xeb2189f734aa831d, 0xf712b443bbd52b7b, 0xa5e9ec7501d523e4, 0x9a6bb0aa55653b2d, + 0x47b233c92125366e, 0xc1069cd4eabe89f8, 0x999ec0bb696e840a, 0xf148440a256e2c76, 0xc00670ea43ca250d, 0x96cd2a865764dbca, 0x380406926a5e5728, 0xbc807527ed3e12bc, + 0xc605083704f5ecf2, 0xeba09271e88d976b, 0xf7864a44c633682e, 0x93445b8731587ea3, 0x7ab3ee6afbe0211d, 0xb8157268fdae9e4c, 0x5960ea05bad82964, 0xe61acf033d1a45df, + 0x6fb92487298e33bd, 0x8fd0c16206306bab, 0xa5d3b6d479f8e056, 0xb3c4f1ba87bc8696, 0x8f48a4899877186c, 0xe0b62e2929aba83c, 0x331acdabfe94de87, 0x8c71dcd9ba0b4925, + 0x9ff0c08b7f1d0b14, 0xaf8e5410288e1b6f, 0x7ecf0ae5ee44dd9, 0xdb71e91432b1a24a, 0xc9e82cd9f69d6150, 0x892731ac9faf056e, 0xbe311c083a225cd2, 0xab70fe17c79ac6ca, + 0x6dbd630a48aaf406, 0xd64d3d9db981787d, 0x92cbbccdad5b108, 0x85f0468293f0eb4e, 0x25bbf56008c58ea5, 0xa76c582338ed2621, 0xaf2af2b80af6f24e, 0xd1476e2c07286faa, + 0x1af5af660db4aee1, 0x82cca4db847945ca, 0x50d98d9fc890ed4d, 0xa37fce126597973c, 0xe50ff107bab528a0, 0xcc5fc196fefd7d0c, 0x1e53ed49a96272c8, 0xff77b1fcbebcdc4f, + 0x25e8e89c13bb0f7a, 0x9faacf3df73609b1, 0x77b191618c54e9ac, 0xc795830d75038c1d, 0xd59df5b9ef6a2417, 0xf97ae3d0d2446f25, 0x4b0573286b44ad1d, 0x9becce62836ac577, + 0x4ee367f9430aec32, 0xc2e801fb244576d5, 0x229c41f793cda73f, 0xf3a20279ed56d48a, 0x6b43527578c1110f, 0x9845418c345644d6, 0x830a13896b78aaa9, 0xbe5691ef416bd60c, + 0x23cc986bc656d553, 0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa8, 0x94b3a202eb1c3f39, 0x7bf7d71432f3d6a9, 0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc53, 0xe858ad248f5c22c9, + 0xd1b3400f8f9cff68, 0x91376c36d99995be, 0x23100809b9c21fa1, 0xb58547448ffffb2d, 0xabd40a0c2832a78a, 0xe2e69915b3fff9f9, 0x16c90c8f323f516c, 0x8dd01fad907ffc3b, + 0xae3da7d97f6792e3, 0xb1442798f49ffb4a, 0x99cd11cfdf41779c, 0xdd95317f31c7fa1d, 0x40405643d711d583, 0x8a7d3eef7f1cfc52, 0x482835ea666b2572, 0xad1c8eab5ee43b66, + 0xda3243650005eecf, 0xd863b256369d4a40, 0x90bed43e40076a82, 0x873e4f75e2224e68, 0x5a7744a6e804a291, 0xa90de3535aaae202, 0x711515d0a205cb36, 0xd3515c2831559a83, + 0xd5a5b44ca873e03, 0x8412d9991ed58091, 0xe858790afe9486c2, 0xa5178fff668ae0b6, 0x626e974dbe39a872, 0xce5d73ff402d98e3, 0xfb0a3d212dc8128f, 0x80fa687f881c7f8e, + 0x7ce66634bc9d0b99, 0xa139029f6a239f72, 0x1c1fffc1ebc44e80, 0xc987434744ac874e, 0xa327ffb266b56220, 0xfbe9141915d7a922, 0x4bf1ff9f0062baa8, 0x9d71ac8fada6c9b5, + 0x6f773fc3603db4a9, 0xc4ce17b399107c22, 0xcb550fb4384d21d3, 0xf6019da07f549b2b, 0x7e2a53a146606a48, 0x99c102844f94e0fb, 0x2eda7444cbfc426d, 0xc0314325637a1939, + 0xfa911155fefb5308, 0xf03d93eebc589f88, 0x793555ab7eba27ca, 0x96267c7535b763b5, 0x4bc1558b2f3458de, 0xbbb01b9283253ca2, 0x9eb1aaedfb016f16, 0xea9c227723ee8bcb, + 0x465e15a979c1cadc, 0x92a1958a7675175f, 0xbfacd89ec191ec9, 0xb749faed14125d36, 0xcef980ec671f667b, 0xe51c79a85916f484, 0x82b7e12780e7401a, 0x8f31cc0937ae58d2, + 0xd1b2ecb8b0908810, 0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa15, 0xdfbdcece67006ac9, 0x67a791e093e1d49a, 0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e0, 0xaecc49914078536d, + 0x58fae9f773886e18, 0xda7f5bf590966848, 0xaf39a475506a899e, 0x888f99797a5e012d, 0x6d8406c952429603, 0xaab37fd7d8f58178, 0xc8e5087ba6d33b83, 0xd5605fcdcf32e1d6, + 0xfb1e4a9a90880a64, 0x855c3be0a17fcd26, 0x5cf2eea09a55067f, 0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481e, 0xd0601d8efc57b08b, 0xf13b94daf124da26, 0x823c12795db6ce57, + 0x76c53d08d6b70858, 0xa2cb1717b52481ed, 0x54768c4b0c64ca6e, 0xcb7ddcdda26da268, 0xa9942f5dcf7dfd09, 0xfe5d54150b090b02, 0xd3f93b35435d7c4c, 0x9efa548d26e5a6e1, + 0xc47bc5014a1a6daf, 0xc6b8e9b0709f109a, 0x359ab6419ca1091b, 0xf867241c8cc6d4c0, 0xc30163d203c94b62, 0x9b407691d7fc44f8, 0x79e0de63425dcf1d, 0xc21094364dfb5636, + 0x985915fc12f542e4, 0xf294b943e17a2bc4, 0x3e6f5b7b17b2939d, 0x979cf3ca6cec5b5a, 0xa705992ceecf9c42, 0xbd8430bd08277231, 0x50c6ff782a838353, 0xece53cec4a314ebd, + 0xa4f8bf5635246428, 0x940f4613ae5ed136, 0x871b7795e136be99, 0xb913179899f68584, 0x28e2557b59846e3f, 0xe757dd7ec07426e5, 0x331aeada2fe589cf, 0x9096ea6f3848984f, + 0x3ff0d2c85def7621, 0xb4bca50b065abe63, 0xfed077a756b53a9, 0xe1ebce4dc7f16dfb, 0xd3e8495912c62894, 0x8d3360f09cf6e4bd, 0x64712dd7abbbd95c, 0xb080392cc4349dec, + 0xbd8d794d96aacfb3, 0xdca04777f541c567, 0xecf0d7a0fc5583a0, 0x89e42caaf9491b60, 0xf41686c49db57244, 0xac5d37d5b79b6239, 0x311c2875c522ced5, 0xd77485cb25823ac7, + 0x7d633293366b828b, 0x86a8d39ef77164bc, 0xae5dff9c02033197, 0xa8530886b54dbdeb, 0xd9f57f830283fdfc, 0xd267caa862a12d66, 0xd072df63c324fd7b, 0x8380dea93da4bc60, + 0x4247cb9e59f71e6d, 0xa46116538d0deb78, 0x52d9be85f074e608, 0xcd795be870516656, 0x67902e276c921f8b, 0x806bd9714632dff6, 0xba1cd8a3db53b6, 0xa086cfcd97bf97f3, + 0x80e8a40eccd228a4, 0xc8a883c0fdaf7df0, 0x6122cd128006b2cd, 0xfad2a4b13d1b5d6c, 0x796b805720085f81, 0x9cc3a6eec6311a63, 0xcbe3303674053bb0, 0xc3f490aa77bd60fc, + 0xbedbfc4411068a9c, 0xf4f1b4d515acb93b, 0xee92fb5515482d44, 0x991711052d8bf3c5, 0x751bdd152d4d1c4a, 0xbf5cd54678eef0b6, 0xd262d45a78a0635d, 0xef340a98172aace4, + 0x86fb897116c87c34, 0x9580869f0e7aac0e, 0xd45d35e6ae3d4da0, 0xbae0a846d2195712, 0x8974836059cca109, 0xe998d258869facd7, 0x2bd1a438703fc94b, 0x91ff83775423cc06, + 0x7b6306a34627ddcf, 0xb67f6455292cbf08, 0x1a3bc84c17b1d542, 0xe41f3d6a7377eeca, 0x20caba5f1d9e4a93, 0x8e938662882af53e, 0x547eb47b7282ee9c, 0xb23867fb2a35b28d, + 0xe99e619a4f23aa43, 0xdec681f9f4c31f31, 0x6405fa00e2ec94d4, 0x8b3c113c38f9f37e, 0xde83bc408dd3dd04, 0xae0b158b4738705e, 0x9624ab50b148d445, 0xd98ddaee19068c76, + 0x3badd624dd9b0957, 0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d6, 0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4c, 0xd47487cc8470652b, 0x7647c3200069671f, 0x84c8d4dfd2c63f3b, + 0x29ecd9f40041e073, 0xa5fb0a17c777cf09, 0xf468107100525890, 0xcf79cc9db955c2cc, 0x7182148d4066eeb4, 0x81ac1fe293d599bf, 0xc6f14cd848405530, 0xa21727db38cb002f, + 0xb8ada00e5a506a7c, 0xca9cf1d206fdc03b, 0xa6d90811f0e4851c, 0xfd442e4688bd304a, 0x908f4a166d1da663, 0x9e4a9cec15763e2e, 0x9a598e4e043287fe, 0xc5dd44271ad3cdba, + 0x40eff1e1853f29fd, 0xf7549530e188c128, 0xd12bee59e68ef47c, 0x9a94dd3e8cf578b9, 0x82bb74f8301958ce, 0xc13a148e3032d6e7, 0xe36a52363c1faf01, 0xf18899b1bc3f8ca1, + 0xdc44e6c3cb279ac1, 0x96f5600f15a7b7e5, 0x29ab103a5ef8c0b9, 0xbcb2b812db11a5de, 0x7415d448f6b6f0e7, 0xebdf661791d60f56, 0x111b495b3464ad21, 0x936b9fcebb25c995, + 0xcab10dd900beec34, 0xb84687c269ef3bfb, 0x3d5d514f40eea742, 0xe65829b3046b0afa, 0xcb4a5a3112a5112, 0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ab, 0xb3f4e093db73a093, + 0x59ed216765690f56, 0xe0f218b8d25088b8, 0x306869c13ec3532c, 0x8c974f7383725573, 0x1e414218c73a13fb, 0xafbd2350644eeacf, 0xe5d1929ef90898fa, 0xdbac6c247d62a583, + 0xdf45f746b74abf39, 0x894bc396ce5da772, 0x6b8bba8c328eb783, 0xab9eb47c81f5114f, 0x66ea92f3f326564, 0xd686619ba27255a2, 0xc80a537b0efefebd, 0x8613fd0145877585, + 0xbd06742ce95f5f36, 0xa798fc4196e952e7, 0x2c48113823b73704, 0xd17f3b51fca3a7a0, 0xf75a15862ca504c5, 0x82ef85133de648c4, 0x9a984d73dbe722fb, 0xa3ab66580d5fdaf5, + 0xc13e60d0d2e0ebba, 0xcc963fee10b7d1b3, 0x318df905079926a8, 0xffbbcfe994e5c61f, 0xfdf17746497f7052, 0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa633, 0xc7caba6e7c5382c8, + 0xfe64a52ee96b8fc0, 0xf9bd690a1b68637b, 0x3dfdce7aa3c673b0, 0x9c1661a651213e2d, 0x6bea10ca65c084e, 0xc31bfa0fe5698db8, 0x486e494fcff30a62, 0xf3e2f893dec3f126, + 0x5a89dba3c3efccfa, 0x986ddb5c6b3a76b7, 0xf89629465a75e01c, 0xbe89523386091465, 0xf6bbb397f1135823, 0xee2ba6c0678b597f, 0x746aa07ded582e2c, 0x94db483840b717ef, + 0xa8c2a44eb4571cdc, 0xba121a4650e4ddeb, 0x92f34d62616ce413, 0xe896a0d7e51e1566, 0x77b020baf9c81d17, 0x915e2486ef32cd60, 0xace1474dc1d122e, 0xb5b5ada8aaff80b8, + 0xd819992132456ba, 0xe3231912d5bf60e6, 0x10e1fff697ed6c69, 0x8df5efabc5979c8f, 0xca8d3ffa1ef463c1, 0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb2, 0xddd0467c64bce4a0, + 0xac7cb3f6d05ddbde, 0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96b, 0xad4ab7112eb3929d, 0x86c16c98d2c953c6, 0xd89d64d57a607744, 0xe871c7bf077ba8b7, 0x87625f056c7c4a8b, + 0x11471cd764ad4972, 0xa93af6c6c79b5d2d, 0xd598e40d3dd89bcf, 0xd389b47879823479, 0x4aff1d108d4ec2c3, 0x843610cb4bf160cb, 0xcedf722a585139ba, 0xa54394fe1eedb8fe, + 0xc2974eb4ee658828, 0xce947a3da6a9273e, 0x733d226229feea32, 0x811ccc668829b887, 0x806357d5a3f525f, 0xa163ff802a3426a8, 0xca07c2dcb0cf26f7, 0xc9bcff6034c13052, + 0xfc89b393dd02f0b5, 0xfc2c3f3841f17c67, 0xbbac2078d443ace2, 0x9d9ba7832936edc0, 0xd54b944b84aa4c0d, 0xc5029163f384a931, 0xa9e795e65d4df11, 0xf64335bcf065d37d, + 0x4d4617b5ff4a16d5, 0x99ea0196163fa42e, 0x504bced1bf8e4e45, 0xc06481fb9bcf8d39, 0xe45ec2862f71e1d6, 0xf07da27a82c37088, 0x5d767327bb4e5a4c, 0x964e858c91ba2655, + 0x3a6a07f8d510f86f, 0xbbe226efb628afea, 0x890489f70a55368b, 0xeadab0aba3b2dbe5, 0x2b45ac74ccea842e, 0x92c8ae6b464fc96f, 0x3b0b8bc90012929d, 0xb77ada0617e3bbcb, + 0x9ce6ebb40173744, 0xe55990879ddcaabd, 0xcc420a6a101d0515, 0x8f57fa54c2a9eab6, 0x9fa946824a12232d, 0xb32df8e9f3546564, 0x47939822dc96abf9, 0xdff9772470297ebd, + 0x59787e2b93bc56f7, 0x8bfbea76c619ef36, 0x57eb4edb3c55b65a, 0xaefae51477a06b03, 0xede622920b6b23f1, 0xdab99e59958885c4, 0xe95fab368e45eced, 0x88b402f7fd75539b, + 0x11dbcb0218ebb414, 0xaae103b5fcd2a881, 0xd652bdc29f26a119, 0xd59944a37c0752a2, 0x4be76d3346f0495f, 0x857fcae62d8493a5, 0x6f70a4400c562ddb, 0xa6dfbd9fb8e5b88e, + 0xcb4ccd500f6bb952, 0xd097ad07a71f26b2, 0x7e2000a41346a7a7, 0x825ecc24c873782f, 0x8ed400668c0c28c8, 0xa2f67f2dfa90563b, 0x728900802f0f32fa, 0xcbb41ef979346bca, + 0x4f2b40a03ad2ffb9, 0xfea126b7d78186bc, 0xe2f610c84987bfa8, 0x9f24b832e6b0f436, 0xdd9ca7d2df4d7c9, 0xc6ede63fa05d3143, 0x91503d1c79720dbb, 0xf8a95fcf88747d94, + 0x75a44c6397ce912a, 0x9b69dbe1b548ce7c, 0xc986afbe3ee11aba, 0xc24452da229b021b, 0xfbe85badce996168, 0xf2d56790ab41c2a2, 0xfae27299423fb9c3, 0x97c560ba6b0919a5, + 0xdccd879fc967d41a, 0xbdb6b8e905cb600f, 0x5400e987bbc1c920, 0xed246723473e3813, 0x290123e9aab23b68, 0x9436c0760c86e30b, 0xf9a0b6720aaf6521, 0xb94470938fa89bce, + 0xf808e40e8d5b3e69, 0xe7958cb87392c2c2, 0xb60b1d1230b20e04, 0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c2, 0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af3, 0xe2280b6c20dd5232, + 0x25c6da63c38de1b0, 0x8d590723948a535f, 0x579c487e5a38ad0e, 0xb0af48ec79ace837, 0x2d835a9df0c6d851, 0xdcdb1b2798182244, 0xf8e431456cf88e65, 0x8a08f0f8bf0f156b, + 0x1b8e9ecb641b58ff, 0xac8b2d36eed2dac5, 0xe272467e3d222f3f, 0xd7adf884aa879177, 0x5b0ed81dcc6abb0f, 0x86ccbb52ea94baea, 0x98e947129fc2b4e9, 0xa87fea27a539e9a5, + 0x3f2398d747b36224, 0xd29fe4b18e88640e, 0x8eec7f0d19a03aad, 0x83a3eeeef9153e89, 0x1953cf68300424ac, 0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd7, 0xcdb02555653131b6, + 0x3792f412cb06794d, 0x808e17555f3ebf11, 0xe2bbd88bbee40bd0, 0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec4, 0xc8de047564d20a8b, 0xf245825a5a445275, 0xfb158592be068d2e, + 0xeed6e2f0f0d56712, 0x9ced737bb6c4183d, 0x55464dd69685606b, 0xc428d05aa4751e4c, 0xaa97e14c3c26b886, 0xf53304714d9265df, 0xd53dd99f4b3066a8, 0x993fe2c6d07b7fab, + 0xe546a8038efe4029, 0xbf8fdb78849a5f96, 0xde98520472bdd033, 0xef73d256a5c0f77c, 0x963e66858f6d4440, 0x95a8637627989aad, 0xdde7001379a44aa8, 0xbb127c53b17ec159, + 0x5560c018580d5d52, 0xe9d71b689dde71af, 0xaab8f01e6e10b4a6, 0x9226712162ab070d, 0xcab3961304ca70e8, 0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d22, 0xe45c10c42a2b3b05, + 0x8cb89a7db77c506a, 0x8eb98a7a9a5b04e3, 0x77f3608e92adb242, 0xb267ed1940f1c61c, 0x55f038b237591ed3, 0xdf01e85f912e37a3, 0x6b6c46dec52f6688, 0x8b61313bbabce2c6, + 0x2323ac4b3b3da015, 0xae397d8aa96c1b77, 0xabec975e0a0d081a, 0xd9c7dced53c72255, 0x96e7bd358c904a21, 0x881cea14545c7575, 0x7e50d64177da2e54, 0xaa242499697392d2, + 0xdde50bd1d5d0b9e9, 0xd4ad2dbfc3d07787, 0x955e4ec64b44e864, 0x84ec3c97da624ab4, 0xbd5af13bef0b113e, 0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58e, 0xcfb11ead453994ba, + 0x67de18eda5814af2, 0x81ceb32c4b43fcf4, 0x80eacf948770ced7, 0xa2425ff75e14fc31, 0xa1258379a94d028d, 0xcad2f7f5359a3b3e, 0x96ee45813a04330, 0xfd87b5f28300ca0d, + 0x8bca9d6e188853fc, 0x9e74d1b791e07e48, 0x775ea264cf55347e, 0xc612062576589dda, 0x95364afe032a819e, 0xf79687aed3eec551, 0x3a83ddbd83f52205, 0x9abe14cd44753b52, + 0xc4926a9672793543, 0xc16d9a0095928a27, 0x75b7053c0f178294, 0xf1c90080baf72cb1, 0x5324c68b12dd6339, 0x971da05074da7bee, 0xd3f6fc16ebca5e04, 0xbce5086492111aea, + 0x88f4bb1ca6bcf585, 0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6, 0x9392ee8e921d5d07, 0x3aff322e62439fd0, 0xb877aa3236a4b449, 0x9befeb9fad487c3, 0xe69594bec44de15b, + 0x4c2ebe687989a9b4, 0x901d7cf73ab0acd9, 0xf9d37014bf60a11, 0xb424dc35095cd80f, 0x538484c19ef38c95, 0xe12e13424bb40e13, 0x2865a5f206b06fba, 0x8cbccc096f5088cb, + 0xf93f87b7442e45d4, 0xafebff0bcb24aafe, 0xf78f69a51539d749, 0xdbe6fecebdedd5be, 0xb573440e5a884d1c, 0x89705f4136b4a597, 0x31680a88f8953031, 0xabcc77118461cefc, + 0xfdc20d2b36ba7c3e, 0xd6bf94d5e57a42bc, 0x3d32907604691b4d, 0x8637bd05af6c69b5, 0xa63f9a49c2c1b110, 0xa7c5ac471b478423, 0xfcf80dc33721d54, 0xd1b71758e219652b, + 0xd3c36113404ea4a9, 0x83126e978d4fdf3b, 0x645a1cac083126ea, 0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4, 0xcccccccccccccccc, 0xcccccccccccccccd, 0x8000000000000000, 0x0, + 0xa000000000000000, 0x0, 0xc800000000000000, 0x0, 0xfa00000000000000, 0x0, 0x9c40000000000000, 0x0, 0xc350000000000000, 0x0, 0xf424000000000000, 0x0, + 0x9896800000000000, 0x0, 0xbebc200000000000, 0x0, 0xee6b280000000000, 0x0, 0x9502f90000000000, 0x0, 0xba43b74000000000, 0x0, 0xe8d4a51000000000, 0x0, + 0x9184e72a00000000, 0x0, 0xb5e620f480000000, 0x0, 0xe35fa931a0000000, 0x0, 0x8e1bc9bf04000000, 0x0, 0xb1a2bc2ec5000000, 0x0, 0xde0b6b3a76400000, 0x0, + 0x8ac7230489e80000, 0x0, 0xad78ebc5ac620000, 0x0, 0xd8d726b7177a8000, 0x0, 0x878678326eac9000, 0x0, 0xa968163f0a57b400, 0x0, 0xd3c21bcecceda100, 0x0, + 0x84595161401484a0, 0x0, 0xa56fa5b99019a5c8, 0x0, 0xcecb8f27f4200f3a, 0x0, 0x813f3978f8940984, 0x4000000000000000, 0xa18f07d736b90be5, 0x5000000000000000, + 0xc9f2c9cd04674ede, 0xa400000000000000, 0xfc6f7c4045812296, 0x4d00000000000000, 0x9dc5ada82b70b59d, 0xf020000000000000, 0xc5371912364ce305, 0x6c28000000000000, + 0xf684df56c3e01bc6, 0xc732000000000000, 0x9a130b963a6c115c, 0x3c7f400000000000, 0xc097ce7bc90715b3, 0x4b9f100000000000, 0xf0bdc21abb48db20, 0x1e86d40000000000, + 0x96769950b50d88f4, 0x1314448000000000, 0xbc143fa4e250eb31, 0x17d955a000000000, 0xeb194f8e1ae525fd, 0x5dcfab0800000000, 0x92efd1b8d0cf37be, 0x5aa1cae500000000, + 0xb7abc627050305ad, 0xf14a3d9e40000000, 0xe596b7b0c643c719, 0x6d9ccd05d0000000, 0x8f7e32ce7bea5c6f, 0xe4820023a2000000, 0xb35dbf821ae4f38b, 0xdda2802c8a800000, + 0xe0352f62a19e306e, 0xd50b2037ad200000, 0x8c213d9da502de45, 0x4526f422cc340000, 0xaf298d050e4395d6, 0x9670b12b7f410000, 0xdaf3f04651d47b4c, 0x3c0cdd765f114000, + 0x88d8762bf324cd0f, 0xa5880a69fb6ac800, 0xab0e93b6efee0053, 0x8eea0d047a457a00, 0xd5d238a4abe98068, 0x72a4904598d6d880, 0x85a36366eb71f041, 0x47a6da2b7f864750, + 0xa70c3c40a64e6c51, 0x999090b65f67d924, 0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d, 0x82818f1281ed449f, 0xbff8f10e7a8921a4, 0xa321f2d7226895c7, 0xaff72d52192b6a0d, + 0xcbea6f8ceb02bb39, 0x9bf4f8a69f764490, 0xfee50b7025c36a08, 0x2f236d04753d5b4, 0x9f4f2726179a2245, 0x1d762422c946590, 0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef5, + 0xf8ebad2b84e0d58b, 0xd2e0898765a7deb2, 0x9b934c3b330c8577, 0x63cc55f49f88eb2f, 0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fb, 0xf316271c7fc3908a, 0x8bef464e3945ef7a, + 0x97edd871cfda3a56, 0x97758bf0e3cbb5ac, 0xbde94e8e43d0c8ec, 0x3d52eeed1cbea317, 0xed63a231d4c4fb27, 0x4ca7aaa863ee4bdd, 0x945e455f24fb1cf8, 0x8fe8caa93e74ef6a, + 0xb975d6b6ee39e436, 0xb3e2fd538e122b44, 0xe7d34c64a9c85d44, 0x60dbbca87196b616, 0x90e40fbeea1d3a4a, 0xbc8955e946fe31cd, 0xb51d13aea4a488dd, 0x6babab6398bdbe41, + 0xe264589a4dcdab14, 0xc696963c7eed2dd1, 0x8d7eb76070a08aec, 0xfc1e1de5cf543ca2, 0xb0de65388cc8ada8, 0x3b25a55f43294bcb, 0xdd15fe86affad912, 0x49ef0eb713f39ebe, + 0x8a2dbf142dfcc7ab, 0x6e3569326c784337, 0xacb92ed9397bf996, 0x49c2c37f07965404, 0xd7e77a8f87daf7fb, 0xdc33745ec97be906, 0x86f0ac99b4e8dafd, 0x69a028bb3ded71a3, + 0xa8acd7c0222311bc, 0xc40832ea0d68ce0c, 0xd2d80db02aabd62b, 0xf50a3fa490c30190, 0x83c7088e1aab65db, 0x792667c6da79e0fa, 0xa4b8cab1a1563f52, 0x577001b891185938, + 0xcde6fd5e09abcf26, 0xed4c0226b55e6f86, 0x80b05e5ac60b6178, 0x544f8158315b05b4, 0xa0dc75f1778e39d6, 0x696361ae3db1c721, 0xc913936dd571c84c, 0x3bc3a19cd1e38e9, + 0xfb5878494ace3a5f, 0x4ab48a04065c723, 0x9d174b2dcec0e47b, 0x62eb0d64283f9c76, 0xc45d1df942711d9a, 0x3ba5d0bd324f8394, 0xf5746577930d6500, 0xca8f44ec7ee36479, + 0x9968bf6abbe85f20, 0x7e998b13cf4e1ecb, 0xbfc2ef456ae276e8, 0x9e3fedd8c321a67e, 0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101e, 0x95d04aee3b80ece5, 0xbba1f1d158724a12, + 0xbb445da9ca61281f, 0x2a8a6e45ae8edc97, 0xea1575143cf97226, 0xf52d09d71a3293bd, 0x924d692ca61be758, 0x593c2626705f9c56, 0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836c, + 0xe498f455c38b997a, 0xb6dfb9c0f956447, 0x8edf98b59a373fec, 0x4724bd4189bd5eac, 0xb2977ee300c50fe7, 0x58edec91ec2cb657, 0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ed, + 0x8b865b215899f46c, 0xbd79e0d20082ee74, 0xae67f1e9aec07187, 0xecd8590680a3aa11, 0xda01ee641a708de9, 0xe80e6f4820cc9495, 0x884134fe908658b2, 0x3109058d147fdcdd, + 0xaa51823e34a7eede, 0xbd4b46f0599fd415, 0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91a, 0x850fadc09923329e, 0x3e2cf6bc604ddb0, 0xa6539930bf6bff45, 0x84db8346b786151c, + 0xcfe87f7cef46ff16, 0xe612641865679a63, 0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07e, 0xa26da3999aef7749, 0xe3be5e330f38f09d, 0xcb090c8001ab551c, 0x5cadf5bfd3072cc5, + 0xfdcb4fa002162a63, 0x73d9732fc7c8f7f6, 0x9e9f11c4014dda7e, 0x2867e7fddcdd9afa, 0xc646d63501a1511d, 0xb281e1fd541501b8, 0xf7d88bc24209a565, 0x1f225a7ca91a4226, + 0x9ae757596946075f, 0x3375788de9b06958, 0xc1a12d2fc3978937, 0x52d6b1641c83ae, 0xf209787bb47d6b84, 0xc0678c5dbd23a49a, 0x9745eb4d50ce6332, 0xf840b7ba963646e0, + 0xbd176620a501fbff, 0xb650e5a93bc3d898, 0xec5d3fa8ce427aff, 0xa3e51f138ab4cebe, 0x93ba47c980e98cdf, 0xc66f336c36b10137, 0xb8a8d9bbe123f017, 0xb80b0047445d4184, + 0xe6d3102ad96cec1d, 0xa60dc059157491e5, 0x9043ea1ac7e41392, 0x87c89837ad68db2f, 0xb454e4a179dd1877, 0x29babe4598c311fb, 0xe16a1dc9d8545e94, 0xf4296dd6fef3d67a, + 0x8ce2529e2734bb1d, 0x1899e4a65f58660c, 0xb01ae745b101e9e4, 0x5ec05dcff72e7f8f, 0xdc21a1171d42645d, 0x76707543f4fa1f73, 0x899504ae72497eba, 0x6a06494a791c53a8, + 0xabfa45da0edbde69, 0x487db9d17636892, 0xd6f8d7509292d603, 0x45a9d2845d3c42b6, 0x865b86925b9bc5c2, 0xb8a2392ba45a9b2, 0xa7f26836f282b732, 0x8e6cac7768d7141e, + 0xd1ef0244af2364ff, 0x3207d795430cd926, 0x8335616aed761f1f, 0x7f44e6bd49e807b8, 0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a6, 0xcd036837130890a1, 0x36dba887c37a8c0f, + 0x802221226be55a64, 0xc2494954da2c9789, 0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6c, 0xc83553c5c8965d3d, 0x6f92829494e5acc7, 0xfa42a8b73abbf48c, 0xcb772339ba1f17f9, + 0x9c69a97284b578d7, 0xff2a760414536efb, 0xc38413cf25e2d70d, 0xfef5138519684aba, 0xf46518c2ef5b8cd1, 0x7eb258665fc25d69, 0x98bf2f79d5993802, 0xef2f773ffbd97a61, + 0xbeeefb584aff8603, 0xaafb550ffacfd8fa, 0xeeaaba2e5dbf6784, 0x95ba2a53f983cf38, 0x952ab45cfa97a0b2, 0xdd945a747bf26183, 0xba756174393d88df, 0x94f971119aeef9e4, + 0xe912b9d1478ceb17, 0x7a37cd5601aab85d, 0x91abb422ccb812ee, 0xac62e055c10ab33a, 0xb616a12b7fe617aa, 0x577b986b314d6009, 0xe39c49765fdf9d94, 0xed5a7e85fda0b80b, + 0x8e41ade9fbebc27d, 0x14588f13be847307, 0xb1d219647ae6b31c, 0x596eb2d8ae258fc8, 0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bb, 0x8aec23d680043bee, 0x25de7bb9480d5854, + 0xada72ccc20054ae9, 0xaf561aa79a10ae6a, 0xd910f7ff28069da4, 0x1b2ba1518094da04, 0x87aa9aff79042286, 0x90fb44d2f05d0842, 0xa99541bf57452b28, 0x353a1607ac744a53, + 0xd3fa922f2d1675f2, 0x42889b8997915ce8, 0x847c9b5d7c2e09b7, 0x69956135febada11, 0xa59bc234db398c25, 0x43fab9837e699095, 0xcf02b2c21207ef2e, 0x94f967e45e03f4bb, + 0x8161afb94b44f57d, 0x1d1be0eebac278f5, 0xa1ba1ba79e1632dc, 0x6462d92a69731732, 0xca28a291859bbf93, 0x7d7b8f7503cfdcfe, 0xfcb2cb35e702af78, 0x5cda735244c3d43e, + 0x9defbf01b061adab, 0x3a0888136afa64a7, 0xc56baec21c7a1916, 0x88aaa1845b8fdd0, 0xf6c69a72a3989f5b, 0x8aad549e57273d45, 0x9a3c2087a63f6399, 0x36ac54e2f678864b, + 0xc0cb28a98fcf3c7f, 0x84576a1bb416a7dd, 0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d5, 0x969eb7c47859e743, 0x9f644ae5a4b1b325, 0xbc4665b596706114, 0x873d5d9f0dde1fee, + 0xeb57ff22fc0c7959, 0xa90cb506d155a7ea, 0x9316ff75dd87cbd8, 0x9a7f12442d588f2, 0xb7dcbf5354e9bece, 0xc11ed6d538aeb2f, 0xe5d3ef282a242e81, 0x8f1668c8a86da5fa, + 0x8fa475791a569d10, 0xf96e017d694487bc, 0xb38d92d760ec4455, 0x37c981dcc395a9ac, 0xe070f78d3927556a, 0x85bbe253f47b1417, 0x8c469ab843b89562, 0x93956d7478ccec8e, + 0xaf58416654a6babb, 0x387ac8d1970027b2, 0xdb2e51bfe9d0696a, 0x6997b05fcc0319e, 0x88fcf317f22241e2, 0x441fece3bdf81f03, 0xab3c2fddeeaad25a, 0xd527e81cad7626c3, + 0xd60b3bd56a5586f1, 0x8a71e223d8d3b074, 0x85c7056562757456, 0xf6872d5667844e49, 0xa738c6bebb12d16c, 0xb428f8ac016561db, 0xd106f86e69d785c7, 0xe13336d701beba52, + 0x82a45b450226b39c, 0xecc0024661173473, 0xa34d721642b06084, 0x27f002d7f95d0190, 0xcc20ce9bd35c78a5, 0x31ec038df7b441f4, 0xff290242c83396ce, 0x7e67047175a15271, + 0x9f79a169bd203e41, 0xf0062c6e984d386, 0xc75809c42c684dd1, 0x52c07b78a3e60868, 0xf92e0c3537826145, 0xa7709a56ccdf8a82, 0x9bbcc7a142b17ccb, 0x88a66076400bb691, + 0xc2abf989935ddbfe, 0x6acff893d00ea435, 0xf356f7ebf83552fe, 0x583f6b8c4124d43, 0x98165af37b2153de, 0xc3727a337a8b704a, 0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5c, + 0xeda2ee1c7064130c, 0x1162def06f79df73, 0x9485d4d1c63e8be7, 0x8addcb5645ac2ba8, 0xb9a74a0637ce2ee1, 0x6d953e2bd7173692, 0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0437, + 0x910ab1d4db9914a0, 0x1d9c9892400a22a2, 0xb54d5e4a127f59c8, 0x2503beb6d00cab4b, 0xe2a0b5dc971f303a, 0x2e44ae64840fd61d, 0x8da471a9de737e24, 0x5ceaecfed289e5d2, + 0xb10d8e1456105dad, 0x7425a83e872c5f47, 0xdd50f1996b947518, 0xd12f124e28f77719, 0x8a5296ffe33cc92f, 0x82bd6b70d99aaa6f, 0xace73cbfdc0bfb7b, 0x636cc64d1001550b, + 0xd8210befd30efa5a, 0x3c47f7e05401aa4e, 0x8714a775e3e95c78, 0x65acfaec34810a71, 0xa8d9d1535ce3b396, 0x7f1839a741a14d0d, 0xd31045a8341ca07c, 0x1ede48111209a050, + 0x83ea2b892091e44d, 0x934aed0aab460432, 0xa4e4b66b68b65d60, 0xf81da84d5617853f, 0xce1de40642e3f4b9, 0x36251260ab9d668e, 0x80d2ae83e9ce78f3, 0xc1d72b7c6b426019, + 0xa1075a24e4421730, 0xb24cf65b8612f81f, 0xc94930ae1d529cfc, 0xdee033f26797b627, 0xfb9b7cd9a4a7443c, 0x169840ef017da3b1, 0x9d412e0806e88aa5, 0x8e1f289560ee864e, + 0xc491798a08a2ad4e, 0xf1a6f2bab92a27e2, 0xf5b5d7ec8acb58a2, 0xae10af696774b1db, 0x9991a6f3d6bf1765, 0xacca6da1e0a8ef29, 0xbff610b0cc6edd3f, 0x17fd090a58d32af3, + 0xeff394dcff8a948e, 0xddfc4b4cef07f5b0, 0x95f83d0a1fb69cd9, 0x4abdaf101564f98e, 0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f1, 0xea53df5fd18d5513, 0x84c86189216dc5ed, + 0x92746b9be2f8552c, 0x32fd3cf5b4e49bb4, 0xb7118682dbb66a77, 0x3fbc8c33221dc2a1, 0xe4d5e82392a40515, 0xfabaf3feaa5334a, 0x8f05b1163ba6832d, 0x29cb4d87f2a7400e, + 0xb2c71d5bca9023f8, 0x743e20e9ef511012, 0xdf78e4b2bd342cf6, 0x914da9246b255416, 0x8bab8eefb6409c1a, 0x1ad089b6c2f7548e, 0xae9672aba3d0c320, 0xa184ac2473b529b1, + 0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741e, 0x8865899617fb1871, 0x7e2fa67c7a658892, 0xaa7eebfb9df9de8d, 0xddbb901b98feeab7, 0xd51ea6fa85785631, 0x552a74227f3ea565, + 0x8533285c936b35de, 0xd53a88958f87275f, 0xa67ff273b8460356, 0x8a892abaf368f137, 0xd01fef10a657842c, 0x2d2b7569b0432d85, 0x8213f56a67f6b29b, 0x9c3b29620e29fc73, + 0xa298f2c501f45f42, 0x8349f3ba91b47b8f, 0xcb3f2f7642717713, 0x241c70a936219a73, 0xfe0efb53d30dd4d7, 0xed238cd383aa0110, 0x9ec95d1463e8a506, 0xf4363804324a40aa, + 0xc67bb4597ce2ce48, 0xb143c6053edcd0d5, 0xf81aa16fdc1b81da, 0xdd94b7868e94050a, 0x9b10a4e5e9913128, 0xca7cf2b4191c8326, 0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f0, + 0xf24a01a73cf2dccf, 0xbc633b39673c8cec, 0x976e41088617ca01, 0xd5be0503e085d813, 0xbd49d14aa79dbc82, 0x4b2d8644d8a74e18, 0xec9c459d51852ba2, 0xddf8e7d60ed1219e, + 0x93e1ab8252f33b45, 0xcabb90e5c942b503, 0xb8da1662e7b00a17, 0x3d6a751f3b936243, 0xe7109bfba19c0c9d, 0xcc512670a783ad4, 0x906a617d450187e2, 0x27fb2b80668b24c5, + 0xb484f9dc9641e9da, 0xb1f9f660802dedf6, 0xe1a63853bbd26451, 0x5e7873f8a0396973, 0x8d07e33455637eb2, 0xdb0b487b6423e1e8, 0xb049dc016abc5e5f, 0x91ce1a9a3d2cda62, + 0xdc5c5301c56b75f7, 0x7641a140cc7810fb, 0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9d, 0xac2820d9623bf429, 0x546345fa9fbdcd44, 0xd732290fbacaf133, 0xa97c177947ad4095, + 0x867f59a9d4bed6c0, 0x49ed8eabcccc485d, 0xa81f301449ee8c70, 0x5c68f256bfff5a74, 0xd226fc195c6a2f8c, 0x73832eec6fff3111, 0x83585d8fd9c25db7, 0xc831fd53c5ff7eab, + 0xa42e74f3d032f525, 0xba3e7ca8b77f5e55, 0xcd3a1230c43fb26f, 0x28ce1bd2e55f35eb, 0x80444b5e7aa7cf85, 0x7980d163cf5b81b3, 0xa0555e361951c366, 0xd7e105bcc332621f, + 0xc86ab5c39fa63440, 0x8dd9472bf3fefaa7, 0xfa856334878fc150, 0xb14f98f6f0feb951, 0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d3, 0xc3b8358109e84f07, 0xa862f80ec4700c8, + 0xf4a642e14c6262c8, 0xcd27bb612758c0fa, 0x98e7e9cccfbd7dbd, 0x8038d51cb897789c, 0xbf21e44003acdd2c, 0xe0470a63e6bd56c3, 0xeeea5d5004981478, 0x1858ccfce06cac74, + 0x95527a5202df0ccb, 0xf37801e0c43ebc8, 0xbaa718e68396cffd, 0xd30560258f54e6ba, 0xe950df20247c83fd, 0x47c6b82ef32a2069, 0x91d28b7416cdd27e, 0x4cdc331d57fa5441, + 0xb6472e511c81471d, 0xe0133fe4adf8e952, 0xe3d8f9e563a198e5, 0x58180fddd97723a6, 0x8e679c2f5e44ff8f, 0x570f09eaa7ea7648 }; + }; + + using powers = powers_template<>; + + template JSONIFIER_INLINE constexpr value_128 computeProductApproximation(int64_t q, uint64_t w) noexcept { + constexpr int32_t bitPrecision{ bitPrecisionNew }; + const int32_t index = 2 * int32_t(q - powers::smallestPowerOfFive); + + value_128 firstproduct = fullMultiplication(w, powers::powerOfFive128[index]); + static_assert((bitPrecision >= 0) && (bitPrecision <= 64), " precision should be in (0,64]"); + constexpr uint64_t precisionMask = (bitPrecision < 64) ? (static_cast(0xFFFFFFFFFFFFFFFF) >> bitPrecision) : static_cast(0xFFFFFFFFFFFFFFFF); + if ((firstproduct.high & precisionMask) == precisionMask) { + value_128 secondproduct = fullMultiplication(w, powers::powerOfFive128[index + 1]); + firstproduct.low += secondproduct.high; + if (secondproduct.high > firstproduct.low) { + firstproduct.high++; + } + } + return firstproduct; + } + + JSONIFIER_INLINE constexpr int32_t power(int32_t q) noexcept { + return (((152170 + 65536) * q) >> 16) + 63; + } + + template JSONIFIER_INLINE constexpr adjusted_mantissa computeFloat(int64_t q, uint64_t w) noexcept { + adjusted_mantissa answer; + if ((w == 0) || (q < binary::smallestPowerOfTen)) { + answer.power2 = 0; + answer.mantissa = 0; + + return answer; + } + if (q > binary::largestPowerOfTen) { + answer.power2 = binary::infinitePower; + answer.mantissa = 0; + return answer; + } + + int32_t lz = static_cast(simd::countlZeroUnsafe(w)); + w <<= lz; + + value_128 product = computeProductApproximation(q, w); + + constexpr auto modifiedMantissaBits{ 64 - binary::mantissaExplicitBits - 3 }; + int32_t upperbit = int32_t(product.high >> 63); + int32_t shift = upperbit + modifiedMantissaBits; + + answer.mantissa = product.high >> shift; + + answer.power2 = int32_t(power(int32_t(q)) + upperbit - lz - binary::minimumExponent); + constexpr auto shifted1{ 1ull << binary::mantissaExplicitBits }; + constexpr auto shifted2{ 2ull << binary::mantissaExplicitBits }; + if (answer.power2 <= 0) { + if (-answer.power2 + 1 >= 64) { + answer.power2 = 0; + answer.mantissa = 0; + + return answer; + } + + answer.mantissa >>= -answer.power2 + 1; + + answer.mantissa += (answer.mantissa & 1); + answer.mantissa >>= 1; + + answer.power2 = (answer.mantissa < (shifted1)) ? 0 : 1; + return answer; + } + + if ((product.low <= 1) && (q >= binary::minExponentRoundToEven) && (q <= binary::maxExponentRoundToEven) && ((answer.mantissa & 3) == 1)) { + if ((answer.mantissa << shift) == product.high) { + answer.mantissa &= ~1ull; + } + } + + answer.mantissa += (answer.mantissa & 1); + answer.mantissa >>= 1; + if (answer.mantissa >= (shifted2)) { + answer.mantissa = (shifted1); + answer.power2++; + } + + answer.mantissa &= ~(shifted1); + if (answer.power2 >= binary::infinitePower) { + answer.power2 = binary::infinitePower; + answer.mantissa = 0; + } + return answer; + } + + template JSONIFIER_INLINE constexpr adjusted_mantissa computeErrorScaled(int64_t q, uint64_t w, int32_t lz) noexcept { + int32_t hilz = int32_t(w >> 63) ^ 1; + adjusted_mantissa answer; + answer.mantissa = w << hilz; + constexpr int32_t bias = binary::mantissaExplicitBits - binary::minimumExponent; + answer.power2 = int32_t(power(int32_t(q)) + bias - hilz - lz - 62 + invalidAmBias); + return answer; + } + + template JSONIFIER_INLINE constexpr adjusted_mantissa computeError(int64_t q, uint64_t w) noexcept { + int32_t lz = static_cast(simd::countlZero(w)); + w <<= lz; + value_128 product = computeProductApproximation(q, w); + return computeErrorScaled(q, product.high, lz); + } + + using limb = uint64_t; + inline constexpr uint64_t limbBits = 64; + inline constexpr uint64_t limbBitsSub1 = limbBits - 1; + + template consteval auto getPowerOfTwo() noexcept { + static_assert(n > 0, "Value must be greater than zero."); + static_assert((n & (n - 1)) == 0, "Value must be a power of two."); + + return simd::countrZero(n); + } + + inline constexpr uint64_t limbBitsPow2 = getPowerOfTwo(); + + using limb_span = span; + + inline constexpr uint64_t bigintBits = 4000; + inline constexpr uint64_t bigintLimbs = bigintBits / limbBits; + + template struct stackvec { + static constexpr uint16_t size{ sizeNew }; + + JSONIFIER_INLINE stackvec() = default; + + JSONIFIER_INLINE constexpr stackvec(limb_span s) noexcept { + tryExtend(s); + } + + stackvec& operator=(stackvec&& other) = delete; + stackvec(stackvec&&) = delete; + stackvec& operator=(const stackvec&) = delete; + stackvec(const stackvec&) = delete; + + JSONIFIER_INLINE constexpr void resizeUnchecked(uint64_t newLen, limb value) noexcept { + if (newLen > length) { + uint64_t count = newLen - length; + limb* first = data + length; + limb* last = first + count; + ::std::fill(first, last, value); + setLen(newLen); + } else { + setLen(newLen); + } + } + + JSONIFIER_INLINE constexpr bool tryResize(uint64_t newLen, limb value) noexcept { + if (newLen > capacity()) { + return false; + } else { + resizeUnchecked(newLen, value); + return true; + } + } + + JSONIFIER_INLINE constexpr const limb& rindex(uint64_t index) const noexcept { + uint64_t rindex = length - index - 1; + return data[rindex]; + } + + JSONIFIER_INLINE constexpr bool nonzero(uint64_t index) const noexcept { + while (index < length) { + if (rindex(index) != 0) { + return true; + } + index++; + } + return false; + } + + JSONIFIER_INLINE constexpr void extendUnchecked(limb_span s) noexcept { + limb* ptr = data + length; + std::copy_n(s.ptr, s.end - s.ptr, ptr); + setLen(length + static_cast(s.end - s.ptr)); + } + + JSONIFIER_INLINE constexpr void pushUnchecked(limb value) noexcept { + data[length] = value; + length++; + } + + JSONIFIER_INLINE constexpr bool tryExtend(limb_span s) noexcept { + if (length + static_cast(s.end - s.ptr) <= capacity()) { + extendUnchecked(s); + return true; + } else { + return false; + } + } + + JSONIFIER_INLINE constexpr void setLen(uint64_t len) noexcept { + length = uint16_t(len); + } + + JSONIFIER_INLINE constexpr uint64_t capacity() const noexcept { + return size; + } + + JSONIFIER_INLINE constexpr bool tryPush(limb value) noexcept { + if (length < capacity()) { + pushUnchecked(value); + return true; + } else { + return false; + } + } + + JSONIFIER_INLINE constexpr bool isEmpty() const noexcept { + return length == 0; + } + + JSONIFIER_INLINE constexpr void normalize() noexcept { + while (length > 0 && rindex(0) == 0) { + length--; + } + } + + limb data[size]; + uint16_t length{}; + }; + + JSONIFIER_INLINE constexpr limb scalarAdd(limb x, limb y, bool& overflow) noexcept { + limb z; + +#if defined(__has_builtin) + #if __has_builtin(__builtin_add_overflow) + overflow = __builtin_add_overflow(x, y, &z); + return z; + #endif +#endif + + z = x + y; + overflow = z < x; + return z; + } + + template JSONIFIER_INLINE constexpr bool smallAddFrom(stackvec& vec, limb y, uint64_t start) noexcept { + uint64_t index = start; + limb carryLocal = y; + bool overflow; + while (carryLocal != 0 && index < vec.length) { + vec.data[index] = scalarAdd(vec.data[index], carryLocal, overflow); + carryLocal = limb(overflow); + index += 1; + } + if (carryLocal != 0) { + if (!vec.tryPush(carryLocal)) { + return false; + } + } + return true; + } + + template JSONIFIER_INLINE constexpr bool largeAddFrom(stackvec& x, limb_span y, uint64_t start) noexcept { + if ((x.length) < start || static_cast(y.end - y.ptr) > (x.length) - start) { + if (!x.tryResize(static_cast((y.end - y.ptr)) + start, 0)) { + return false; + } + } + + bool carryLocal = false; + for (uint64_t index = 0; index < static_cast(y.end - y.ptr); index++) { + limb xi = x.data[index + start]; + limb yi = y.ptr[index]; + bool c1 = false; + bool c2 = false; + xi = scalarAdd(xi, yi, c1); + if (carryLocal) { + xi = scalarAdd(xi, 1, c2); + } + x.data[index + start] = xi; + carryLocal = c1 || c2; + } + + if (carryLocal) { + if (!smallAddFrom(x, 1ULL, static_cast(y.end - y.ptr) + start)) { + return false; + } + } + return true; + } + + template JSONIFIER_INLINE constexpr bool largeAddFrom(stackvec& x, limb_span y) noexcept { + return largeAddFrom(x, y, 0); + } + + template JSONIFIER_INLINE constexpr bool smallAdd(stackvec& vec, limb y) noexcept { + return smallAddFrom(vec, y, 0); + } + + JSONIFIER_INLINE uint64_t uint64Hi64(uint64_t r0, bool& truncated) noexcept { + truncated = false; + int32_t shl = static_cast(simd::countlZero(r0)); + return r0 << shl; + } + + JSONIFIER_INLINE uint64_t uint64Hi64(uint64_t r0, uint64_t r1, bool& truncated) noexcept { + int32_t shl = static_cast(simd::countlZero(r0)); + if (shl == 0) { + truncated = r1 != 0; + return r0; + } else { + int32_t shr = 64 - shl; + truncated = (r1 << shl) != 0; + return (r0 << shl) | (r1 >> shr); + } + } + + JSONIFIER_INLINE limb scalarMul(limb x, limb y, limb& carryLocal) noexcept { +#if defined(__SIZEOF_INT128__) + + __uint128_t z = __uint128_t(x) * __uint128_t(y) + __uint128_t(carryLocal); + carryLocal = limb(z >> limbBits); + return limb(z); +#else + value_128 z = fullMultiplication(x, y); + bool overflow; + z.low = scalarAdd(z.low, carryLocal, overflow); + z.high += static_cast(overflow); + carryLocal = z.high; + return z.low; +#endif + } + + template JSONIFIER_INLINE constexpr bool smallMul(stackvec& vec, limb y) noexcept { + limb carryLocal = 0; + for (uint64_t index = 0; index < vec.length; index++) { + vec.data[index] = scalarMul(vec.data[index], y, carryLocal); + } + if (carryLocal != 0) { + if (!vec.tryPush(carryLocal)) { + return false; + } + } + return true; + } + + template JSONIFIER_INLINE constexpr bool longMul(stackvec& x, limb_span y) noexcept { + limb_span xs = limb_span(x.data, x.data + x.length); + stackvec z(xs); + limb_span zs = limb_span(z.data, z.data + z.length); + + if ((y.end - y.ptr) != 0) { + limb y0 = y.ptr[0]; + if (!smallMul(x, y0)) { + return false; + } + for (uint64_t index = 1; index < static_cast(y.end - y.ptr); index++) { + limb yi = y.ptr[index]; + stackvec zi; + if (yi != 0) { + zi.setLen(0); + if (!zi.tryExtend(zs)) { + return false; + } + if (!smallMul(zi, yi)) { + return false; + } + limb_span zis = limb_span(zi.data, zi.data + zi.length); + if (!largeAddFrom(x, zis, index)) { + return false; + } + } + } + } + + x.normalize(); + return true; + } + + template JSONIFIER_INLINE constexpr bool largeMul(stackvec& x, limb_span y) noexcept { + if ((y.end - y.ptr) == 1) { + if (!smallMul(x, y.ptr[0])) { + return false; + } + } else { + if (!longMul(x, y)) { + return false; + } + } + return true; + } + + JSONIFIER_INLINE constexpr uint64_t emptyHi64(bool& truncated) noexcept { + truncated = false; + return 0; + } + + template struct pow5_tables { + static constexpr uint32_t largeStep = 135; + alignas(64) static constexpr uint64_t smallPowerOf5[]{ 1UL, 5UL, 25UL, 125UL, 625UL, 3125UL, 15625UL, 78125UL, 390625UL, 1953125UL, 9765625UL, 48828125UL, 244140625UL, + 1220703125UL, 6103515625UL, 30517578125UL, 152587890625UL, 762939453125UL, 3814697265625UL, 19073486328125UL, 95367431640625UL, 476837158203125UL, 2384185791015625UL, + 11920928955078125UL, 59604644775390625UL, 298023223876953125UL, 1490116119384765625UL, 7450580596923828125UL }; + alignas(64) static constexpr limb largePowerOf5[]{ 1414648277510068013UL, 9180637584431281687UL, 4539964771860779200UL, 10482974169319127550UL, 198276706040285095UL }; + }; + + struct bigint : pow5_tables<> { + JSONIFIER_INLINE constexpr bigint() noexcept { + } + + JSONIFIER_INLINE bigint(uint64_t value) noexcept { + vec.pushUnchecked(value); + vec.normalize(); + } + + bigint& operator=(bigint&& other) = delete; + bigint(bigint&&) = delete; + bigint& operator=(const bigint&) = delete; + bigint(const bigint&) = delete; + + JSONIFIER_INLINE constexpr bool pow5(uint32_t exp) noexcept { + constexpr uint64_t largeLength = sizeof(largePowerOf5) / sizeof(limb); + constexpr limb_span large = limb_span(largePowerOf5, largePowerOf5 + largeLength); + while (exp >= largeStep) { + if (!largeMul(vec, large)) { + return false; + } + exp -= largeStep; + } + constexpr uint32_t smallStep = 27; + constexpr limb maxNative = 7450580596923828125UL; + while (exp >= smallStep) { + if (!smallMul(vec, maxNative)) { + return false; + } + exp -= smallStep; + } + if (exp != 0) { + if (!smallMul(vec, limb((static_cast(smallPowerOf5[0]), smallPowerOf5[exp])))) { + return false; + } + } + + return true; + } + + JSONIFIER_INLINE constexpr int32_t ctlz() const noexcept { + if (vec.isEmpty()) { + return 0; + } else { + return static_cast(simd::countlZero(vec.rindex(0))); + } + } + + JSONIFIER_INLINE constexpr int32_t compare(const bigint& other) const noexcept { + if (vec.length > other.vec.length) { + return 1; + } else if (vec.length < other.vec.length) { + return -1; + } else { + for (uint64_t index = vec.length; index > 0; index--) { + limb xi = vec.data[index - 1]; + limb yi = other.vec.data[index - 1]; + if (xi > yi) { + return 1; + } else if (xi < yi) { + return -1; + } + } + return 0; + } + } + + JSONIFIER_INLINE constexpr uint64_t hi64(bool& truncated) const noexcept { + if (vec.length == 0) { + return emptyHi64(truncated); + } else if (vec.length == 1) { + return uint64Hi64(vec.rindex(0), truncated); + } else { + uint64_t result = uint64Hi64(vec.rindex(0), vec.rindex(1), truncated); + truncated |= vec.nonzero(2); + return result; + } + } + + JSONIFIER_INLINE constexpr bool shlBits(uint64_t num) noexcept { + uint64_t shl = num; + uint64_t shr = limbBits - shl; + limb prev = 0; + for (uint64_t index = 0; index < vec.length; index++) { + limb xi = vec.data[index]; + vec.data[index] = (xi << shl) | (prev >> shr); + prev = xi; + } + + limb carryLocal = prev >> shr; + if (carryLocal != 0) { + return vec.tryPush(carryLocal); + } + return true; + } + + JSONIFIER_INLINE constexpr int32_t bitLength() const noexcept { + int32_t lz = ctlz(); + return int32_t(limbBits * vec.length) - lz; + } + + JSONIFIER_INLINE constexpr bool pow10(uint32_t exp) noexcept { + if (!pow5(exp)) { + return false; + } + return pow2(exp); + } + + JSONIFIER_INLINE constexpr bool pow2(uint32_t exp) noexcept { + return shl(exp); + } + + JSONIFIER_INLINE bool shlLimbs(uint64_t num) noexcept { + if (num + vec.length > vec.capacity()) { + return false; + } else if (!vec.isEmpty()) { + limb* dst = vec.data + num; + const limb* src = vec.data; + std::copy_backward(src, src + vec.length, dst + vec.length); + + limb* first = vec.data; + limb* last = first + num; + ::std::fill(first, last, static_cast(0)); + vec.setLen(num + vec.length); + return true; + } else { + return true; + } + } + + JSONIFIER_INLINE constexpr bool shl(uint64_t num) noexcept { + uint64_t rem = num & limbBitsSub1; + uint64_t div = num >> limbBitsPow2; + if (rem != 0) { + if (!shlBits(rem)) { + return false; + } + } + if (div != 0) { + if (!shlLimbs(div)) { + return false; + } + } + return true; + } + + JSONIFIER_INLINE constexpr bool mul(limb y) noexcept { + return smallMul(vec, y); + } + + JSONIFIER_INLINE constexpr bool add(limb y) noexcept { + return smallAdd(vec, y); + } + + stackvec vec; + }; + + alignas(64) inline constexpr uint64_t powersOfTenUint64[]{ 1UL, 10UL, 100UL, 1000UL, 10000UL, 100000UL, 1000000UL, 10000000UL, 100000000UL, 1000000000UL, 10000000000UL, + 100000000000UL, 1000000000000UL, 10000000000000UL, 100000000000000UL, 1000000000000000UL, 10000000000000000UL, 100000000000000000UL, 1000000000000000000UL, + 10000000000000000000UL }; + + template JSONIFIER_INLINE constexpr void round(adjusted_mantissa& am, extra_types... extra) noexcept { + constexpr int32_t mantissaShift = 64 - binary_format::mantissaExplicitBits - 1; + constexpr auto shifted2{ 2ull << binary_format::mantissaExplicitBits }; + constexpr auto shifted1{ 1ull << binary_format::mantissaExplicitBits }; + if (-am.power2 >= mantissaShift) { + int32_t shift = -am.power2 + 1; + callback::impl(am, std::min(shift, 64), extra...); + am.power2 = (am.mantissa < (shifted1)) ? 0 : 1; + return; + } + + callback::impl(am, mantissaShift, extra...); + + if (am.mantissa >= shifted2) { + am.mantissa = (shifted1); + am.power2++; + } + + am.mantissa &= ~(shifted1); + if (am.power2 >= binary_format::infinitePower) { + am.power2 = binary_format::infinitePower; + am.mantissa = 0; + } + } + + template JSONIFIER_INLINE constexpr void roundNearestTieEven(adjusted_mantissa& am, int32_t shift, extra_type extra) noexcept { + const uint64_t mask = (shift == 64) ? UINT64_MAX : (1ull << shift) - 1; + const uint64_t halfway = (shift == 0) ? 0 : 1ull << (shift - 1); + uint64_t truncatedBits = am.mantissa & mask; + bool isAbove = truncatedBits > halfway; + bool isHalfway = truncatedBits == halfway; + + if (shift == 64) { + am.mantissa = 0; + } else { + am.mantissa >>= shift; + } + am.power2 += shift; + + bool isOdd = (am.mantissa & 1) == 1; + am.mantissa += static_cast(callback::impl(isOdd, isHalfway, isAbove, extra)); + } + + template JSONIFIER_INLINE constexpr bool isTruncated(char_types const* first, char_types const* last) noexcept { + constexpr auto cmpLength{ intCmpLen }; + constexpr auto cmpZeros{ intCmpZeros }; + uint64_t val; + while (last - first >= cmpLength) { + pow2MemcpyWrapper(&val, first); + if (val != cmpZeros) { + return true; + } + first += cmpLength; + } + while (first != last) { + if (*first != char_types('0')) { + return true; + } + ++first; + } + return false; + } + + template JSONIFIER_INLINE constexpr void skipZeros(char_types const*& first, char_types const* last) noexcept { + constexpr auto cmpLength{ intCmpLen }; + constexpr auto cmpZeros{ intCmpZeros }; + uint64_t val; + while (last - first >= cmpLength) { + pow2MemcpyWrapper(&val, first); + if (val != cmpZeros) { + break; + } + first += cmpLength; + } + while (first != last) { + if (*first != char_types('0')) { + break; + } + first++; + } + } + + template JSONIFIER_INLINE constexpr adjusted_mantissa toExtended(value_type value) noexcept { + using equiv_uint = typename binary_format::equiv_uint; + + adjusted_mantissa am; + constexpr int32_t bias = binary_format::mantissaExplicitBits - binary_format::minimumExponent; + equiv_uint bits; + bits = std::bit_cast(value); + if ((bits & binary_format::exponentMask) == 0) { + am.power2 = 1 - bias; + am.mantissa = bits & binary_format::mantissaMask; + } else { + am.power2 = int32_t((bits & binary_format::exponentMask) >> binary_format::mantissaExplicitBits); + am.power2 -= bias; + am.mantissa = (bits & binary_format::mantissaMask) | binary_format::hiddenBitMask; + } + + return am; + } + + template JSONIFIER_INLINE constexpr adjusted_mantissa toExtendedHalfway(value_type value) noexcept { + adjusted_mantissa am = toExtended(value); + am.mantissa <<= 1; + am.mantissa += 1; + am.power2 -= 1; + return am; + } + + JSONIFIER_INLINE constexpr int64_t scientificExponent(uint64_t mantissa, int64_t exponent) noexcept { + while (mantissa >= 10000) { + mantissa /= 10000; + exponent += 4; + } + while (mantissa >= 100) { + mantissa /= 100; + exponent += 2; + } + while (mantissa >= 10) { + mantissa /= 10; + exponent += 1; + } + return exponent; + } + + JSONIFIER_INLINE constexpr void roundDown(adjusted_mantissa& am, int32_t shift) noexcept { + if (shift == 64) { + am.mantissa = 0; + } else { + am.mantissa >>= shift; + } + am.power2 += shift; + } + + template JSONIFIER_INLINE constexpr void parseEightDigits(const char_types*& p, limb& value, uint64_t& counter, uint64_t& count) noexcept { + value = value * 100000000 + parseEightDigitsUnrolled(read8ToU64(p)); + p += 8; + counter += 8; + count += 8; + } + + template JSONIFIER_INLINE constexpr void parseOneDigit(char_types const*& p, limb& value, uint64_t& counter, uint64_t& count) noexcept { + value = value * 10 + limb(*p - char_types('0')); + p++; + counter++; + count++; + } + + template JSONIFIER_INLINE constexpr bool isTruncated(span s) noexcept { + return isTruncated(s.ptr, s.end); + } + + JSONIFIER_INLINE constexpr void addNative(bigint& big, limb power, limb value) noexcept { + big.mul(power); + big.add(value); + } + + JSONIFIER_INLINE constexpr void roundUpBigint(bigint& big, uint64_t& count) noexcept { + addNative(big, 10, 1); + count++; + } + + template + JSONIFIER_INLINE constexpr void parseMantissa(bigint& result, span& integer, span& fraction, uint64_t& digits) noexcept { + constexpr uint64_t maxDigits{ maxDigitsNew }; + + uint64_t counter = 0; + digits = 0; + limb value = 0; + constexpr uint64_t step = 19; + + char_types const* p = integer.ptr; + skipZeros(p, integer.end); + + while (p != integer.end) { + while ((integer.end - p >= 8) && (step - counter >= 8) && (maxDigits - digits >= 8)) { + parseEightDigits(p, value, counter, digits); + } + while (counter < step && p != integer.end && digits < maxDigits) { + parseOneDigit(p, value, counter, digits); + } + if (digits == maxDigits) { + addNative(result, limb(powersOfTenUint64[counter]), value); + bool truncated = isTruncated(p, integer.end); + if (fraction.ptr != nullptr) { + truncated |= isTruncated(fraction); + } + if (truncated) { + roundUpBigint(result, digits); + } + return; + } else { + addNative(result, limb(powersOfTenUint64[counter]), value); + counter = 0; + value = 0; + } + } + + if (fraction.ptr != nullptr) { + p = fraction.ptr; + if (digits == 0) { + skipZeros(p, fraction.end); + } + + while (p != fraction.end) { + while ((fraction.end - p >= 8) && (step - counter >= 8) && (maxDigits - digits >= 8)) { + parseEightDigits(p, value, counter, digits); + } + while (counter < step && p != fraction.end && digits < maxDigits) { + parseOneDigit(p, value, counter, digits); + } + if (digits == maxDigits) { + addNative(result, limb(powersOfTenUint64[counter]), value); + bool truncated = isTruncated(p, fraction.end); + if (truncated) { + roundUpBigint(result, digits); + } + return; + } else { + addNative(result, limb(powersOfTenUint64[counter]), value); + counter = 0; + value = 0; + } + } + } + + if (counter != 0) { + addNative(result, limb(powersOfTenUint64[counter]), value); + } + } + + struct round_down_functor { + JSONIFIER_INLINE static constexpr void impl(adjusted_mantissa& am, int32_t shift) noexcept { + roundDown(am, shift); + } + }; + + struct positive_tie_even_functor { + JSONIFIER_INLINE static constexpr bool impl(bool isOdd, bool isHalfway, bool isAbove, bool truncated) noexcept { + return isAbove || (isHalfway && truncated) || (isOdd && isHalfway); + } + }; + + struct positive_round_functor { + JSONIFIER_INLINE static constexpr void impl(adjusted_mantissa& am, int32_t shift, bool truncated) noexcept { + roundNearestTieEven(am, shift, truncated); + } + }; + + struct negative_tie_even_functor { + JSONIFIER_INLINE static constexpr bool impl(bool isOdd, bool, bool, int32_t ord) noexcept { + if (ord > 0) { + return true; + } else if (ord < 0) { + return false; + } else { + return isOdd; + } + } + }; + + struct negative_round_functor { + JSONIFIER_INLINE static constexpr void impl(adjusted_mantissa& am, int32_t shift, int32_t ord) noexcept { + roundNearestTieEven(am, shift, ord); + } + }; + + template JSONIFIER_INLINE constexpr adjusted_mantissa negativeDigitComp(bigint& bigmant, adjusted_mantissa am, int32_t exponent) noexcept { + bigint& realDigits = bigmant; + int32_t realExp = exponent; + + adjusted_mantissa amB = am; + + round(amB); + value_type b; + toFloat(false, amB, b); + adjusted_mantissa theor = toExtendedHalfway(b); + bigint theorDigits(theor.mantissa); + int32_t theorExp = theor.power2; + + int32_t pow2Exp = theorExp - realExp; + uint32_t pow5Exp = static_cast(-realExp); + if (pow5Exp != 0) { + theorDigits.pow5(pow5Exp); + } + if (pow2Exp > 0) { + theorDigits.pow2(static_cast(pow2Exp)); + } else if (pow2Exp < 0) { + realDigits.pow2(static_cast(-pow2Exp)); + } + + int32_t ord = realDigits.compare(theorDigits); + adjusted_mantissa answer = am; + round(answer, ord); + return answer; + } + + template JSONIFIER_INLINE constexpr adjusted_mantissa positiveDigitComp(bigint& bigmant, int32_t exponent) noexcept { + bigmant.pow10(static_cast(exponent)); + adjusted_mantissa answer; + bool truncated; + answer.mantissa = bigmant.hi64(truncated); + constexpr int32_t bias = binary_format::mantissaExplicitBits - binary_format::minimumExponent; + answer.power2 = bigmant.bitLength() - 64 + bias; + + round(answer, truncated); + + return answer; + } + + template JSONIFIER_INLINE constexpr adjusted_mantissa digitComp(span& integer, span& fraction, + uint64_t mantissa, int64_t exponent, adjusted_mantissa am) noexcept { + am.power2 -= invalidAmBias; + + int32_t sciExp = static_cast(scientificExponent(mantissa, exponent)); + uint64_t digits = 0; + bigint bigmant; + parseMantissa::maxDigits>(bigmant, integer, fraction, digits); + + int32_t exponentNew = sciExp + 1 - int32_t(digits); + if (exponentNew >= 0) { + return positiveDigitComp(bigmant, exponentNew); + } else { + return negativeDigitComp(bigmant, am, exponentNew); + } + } + + JSONIFIER_INLINE bool roundsToNearestFn() noexcept { +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + return false; +#endif + + static volatile float fmin = std::numeric_limits::min(); + volatile float fmini = fmin; + +#if JSONIFIER_COMPILER_MSVC + #pragma warning(push) + +#elif JSONIFIER_COMPILER_CLANG + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wfloat-equal" +#elif JSONIFIER_COMPILER_GCC + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + const float lhs = fmini + 1.0f; + const float rhs = 1.0f - fmini; + return lhs == rhs; +#if JSONIFIER_COMPILER_MSVC + #pragma warning(pop) +#elif JSONIFIER_COMPILER_CLANG + #pragma clang diagnostic pop +#elif JSONIFIER_COMPILER_GCC + #pragma GCC diagnostic pop +#endif + } + +#if JSONIFIER_COMPILER_CLANG + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wglobal-constructors" +#endif + struct rounds_to_nearest { + inline static const bool roundsToNearest{ roundsToNearestFn() }; + }; +#if JSONIFIER_COMPILER_CLANG + #pragma clang diagnostic pop +#endif + +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/forward.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/forward.hpp new file mode 100644 index 000000000000..9c73917d2666 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/forward.hpp @@ -0,0 +1,127 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/forward.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::internal { + + // from + // https://stackoverflow.com/questions/16337610/how-to-know-if-a-type-is-a-specialization-of-stdvector + template typename> inline constexpr bool isSpecialization = false; + + template typename value_type, typename... arg_types> inline constexpr bool isSpecialization, value_type> = true; + + template struct remove_const { + using type = value_type; + }; + + template struct remove_const { + using type = value_type; + }; + + template using remove_const_t = typename remove_const::type; + + template struct remove_volatile { + using type = value_type; + }; + + template struct remove_volatile { + using type = value_type; + }; + + template using remove_volatile_t = typename remove_volatile::type; + + template struct remove_cv { + using type = value_type; + }; + + template struct remove_cv { + using type = value_type; + }; + + template struct remove_cv { + using type = value_type; + }; + + template struct remove_cv { + using type = value_type; + }; + + template using remove_cv_t = typename remove_cv::type; + + template struct remove_pointer { + using type = value_type; + }; + + template struct remove_pointer { + using type = value_type; + }; + + template using remove_pointer_t = typename remove_pointer::type; + + template struct remove_reference { + using type = value_type; + }; + + template struct remove_reference { + using type = value_type; + }; + + template struct remove_reference { + using type = value_type; + }; + + template using remove_reference_t = typename remove_reference::type; + + template using remove_cvref_t = remove_cv_t>; + + template struct conditional; + + template struct conditional { + using type = type01; + }; + + template struct conditional { + using type = type02; + }; + + template using conditional_t = conditional::type; + + template struct parser; + + enum class avx_types : uint8_t { m128 = 0, m256 = 1, m512 = 2 }; + + template struct simd_type_wrapper; + + template<> struct simd_type_wrapper { + static constexpr auto avx_type = avx_types::m128; + using type = jsonifier_simd_int_128; + }; + + template<> struct simd_type_wrapper { + static constexpr auto avx_type = avx_types::m256; + using type = jsonifier_simd_int_256; + }; + + template<> struct simd_type_wrapper { + static constexpr auto avx_type = avx_types::m512; + using type = jsonifier_simd_int_512; + }; + + template struct unrestricter { + using type = value_type; + }; + + template struct unrestricter { + using type = value_type*; + }; + + template using unrestrict_t = typename unrestricter::type; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/get_enum_name.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/get_enum_name.hpp new file mode 100644 index 000000000000..b4deddb2e8e8 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/get_enum_name.hpp @@ -0,0 +1,106 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/get_enum_name.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::internal { + + template consteval string_view getEnumName() noexcept { + string_view str = std::source_location::current().function_name(); +#if JSONIFIER_COMPILER_GCC + str = str.substr(str.find("=") + 2); + uint64_t end = str.find(';'); + str = str.substr(0, end); + uint64_t start = str.findLastOf(':') + 1; + return str.substr(start); +#else + #if JSONIFIER_COMPILER_MSVC + constexpr const char* prettyFunctionTailLocal{ ">(void)" }; + #elif JSONIFIER_COMPILER_CLANG + constexpr const char* prettyFunctionTailLocal{ "]" }; + #endif + str = str.substr(str.find("=") + 2); + uint64_t start = str.findLastOf(':') + 1; + uint64_t end = str.find(prettyFunctionTailLocal); + return str.substr(start, end - start); +#endif + } + + consteval bool isValidEnumName(string_view name) noexcept { + if (name.empty()) { + return false; + } + if (name.findFirstOf("()") != string_view::npos) { + return false; + } + if (name[0] >= '0' && name[0] <= '9') { + return false; + } + return true; + } + + template struct enum_entry_getter; + + template struct enum_entry_getter, current_type> { + template struct entries { + array values{}; + array names{}; + }; + + static consteval uint64_t countValid() noexcept { + uint64_t total{}; + ((isValidEnumName(getEnumName(indices)>()) ? ++total : total), ...); + return total; + } + + template static consteval entries buildEntries() noexcept { + entries result{}; + uint64_t pos{}; + ( + [&] { + constexpr auto value = static_cast(indices); + constexpr string_view name = getEnumName(); + if constexpr (isValidEnumName(name)) { + result.names[pos] = name; + result.values[pos] = value; + ++pos; + } + }(), + ...); + return result; + } + }; + + template(current_type::count)> struct enum_data { + using entry_getter = enum_entry_getter, current_type>; + static constexpr uint64_t validCount{ entry_getter::countValid() }; + static constexpr auto value{ entry_getter::template buildEntries() }; + }; + + template JSONIFIER_INLINE constexpr string_view getName(enum_type type) noexcept { + constexpr auto& entries{ enum_data::value }; + for (uint64_t i = 0; i < entries.values.size(); ++i) { + if (entries.values[i] == type) { + return entries.names[i]; + } + } + return "Unknown Type"; + } + + template JSONIFIER_INLINE basic_stream& operator<<(basic_stream& os, enum_type type) noexcept { + os << getName(type).operator std::basic_string_view(); + return os; + } + + template JSONIFIER_INLINE std::ostream& operator<<(std::ostream& os, enum_type type) noexcept { + os << getName(type).operator std::basic_string_view(); + return os; + } + +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/i_to_str.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/i_to_str.hpp new file mode 100644 index 000000000000..441a50967197 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/i_to_str.hpp @@ -0,0 +1,377 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/i_to_str.hpp + */ +#pragma once + +#include + +namespace jsonifier::internal { + + template consteval uint64_t genTableSize() noexcept { + uint64_t result = 1; + for (uint64_t j = 0; j < count; ++j) { + result *= 10; + } + return result; + } + + template using gen_table_value_t = conditional_t<(count <= 2ULL), uint16_t, uint32_t>; + + template consteval array, genTableSize()> genTable() noexcept { + using value_type = gen_table_value_t; + constexpr uint64_t size = genTableSize(); + array t{}; + for (uint64_t i = 0; i < size; ++i) { + uint64_t divisor = size / 10; + for (uint64_t j = 0; j < count; ++j) { + const auto digit = static_cast(static_cast('0') + (i / divisor % 10)); + const uint64_t shift = (std::endian::native == std::endian::little ? j : sizeof(value_type) - 1 - j) * 8; + t[i] |= static_cast(digit << shift); + divisor /= 10; + } + } + return t; + } + + alignas(64) inline constexpr auto table02{ genTable<2ULL>() }; + inline constexpr const auto* __restrict charTable2DigitData{ table02.data() }; + alignas(64) inline constexpr auto table03{ genTable<3ULL>() }; + inline constexpr const auto* __restrict charTable3DigitData{ table03.data() }; + alignas(64) inline constexpr auto table04{ genTable<4ULL>() }; + inline constexpr const auto* __restrict charTable4DigitData{ table04.data() }; + +#if !JSONIFIER_HAS_INT128 && !(JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_X64) + + template JSONIFIER_INLINE v_type_new mulhiPortable(v_type_new a, v_type_new b) noexcept { + using v_type = v_type_new; + static constexpr uint64_t totalBits = sizeof(v_type_new) * 8; + static constexpr uint64_t halfBits = totalBits / 2; + static constexpr v_type mask = (static_cast(1) << halfBits) - 1; + const v_type aLo = static_cast(a) & mask; + const v_type aHi = static_cast(a) >> halfBits; + const v_type bLo = static_cast(b) & mask; + const v_type bHi = static_cast(b) >> halfBits; + const v_type loLo = aLo * bLo; + const v_type hiLo = aHi * bLo; + const v_type loHi = aLo * bHi; + const v_type hiHi = aHi * bHi; + const v_type cross = (loLo >> halfBits) + (hiLo & mask) + (loHi & mask); + return static_cast(hiHi + (hiLo >> halfBits) + (loHi >> halfBits) + (cross >> halfBits)); + } + +#endif + + struct multiply_and_shift { + JSONIFIER_INLINE static uint64_t impl(uint64_t value) noexcept { +#if JSONIFIER_HAS_INT128 + return static_cast(static_cast<__uint128_t>(value) * 12379400392853802749ULL >> 90); +#elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_X64 + uint64_t highPart; + _umul128(12379400392853802749ULL, value, &highPart); + return static_cast(highPart >> (90 - 64ULL)); +#else + return static_cast(mulhiPortable(value, 12379400392853802749ULL) >> (90 - 64ULL)); +#endif + } + }; + + template JSONIFIER_INLINE void copy3Digits(write_buffer_ptr buf, const v_type value) noexcept { + uint32_t packed; + pow2MemcpyWrapper<4ULL>(&packed, &charTable3DigitData[value]); + pow2MemcpyWrapper<2ULL>(buf, &packed); + buf[2] = static_cast(packed >> 16); + } + + template struct to_chars_internal; + + template struct to_chars; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type a{ static_cast(value * 3518437209ULL >> 45) }; + *buf = static_cast(a + '0'); + pow2MemcpyWrapper<4ULL>(buf + 1, charTable4DigitData + value - a * 10000); + return buf + 5; + } + }; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type ab{ static_cast(value * 3518437209ULL >> 45) }; + pow2MemcpyWrapper<2ULL>(buf, charTable2DigitData + ab); + pow2MemcpyWrapper<4ULL>(buf + 2, charTable4DigitData + value - (ab * 10000ULL)); + return buf + 6; + } + }; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type abc{ static_cast(value * 3518437209ULL >> 45) }; + copy3Digits(buf, abc); + pow2MemcpyWrapper<4ULL>(buf + 3, charTable4DigitData + value - (abc * 10000ULL)); + return buf + 7; + } + }; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type abcd{ static_cast(value * 3518437209ULL >> 45) }; + pow2MemcpyWrapper<4ULL>(buf, charTable4DigitData + abcd); + pow2MemcpyWrapper<4ULL>(buf + 4, charTable4DigitData + value - (abcd * 10000ULL)); + return buf + 8; + } + }; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type a{ static_cast(multiply_and_shift::impl(value)) }; + const v_type bcdefghi{ static_cast(value - a * 100000000ULL) }; + const v_type bcde{ static_cast(bcdefghi * 3518437209ULL >> 45) }; + *buf = static_cast(a + '0'); + pow2MemcpyWrapper<4ULL>(buf + 1, charTable4DigitData + bcde); + pow2MemcpyWrapper<4ULL>(buf + 5, charTable4DigitData + bcdefghi - bcde * 10000ULL); + return buf + 9; + } + }; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type ab{ static_cast(multiply_and_shift::impl(value)) }; + const v_type cdefghij{ static_cast(value - ab * 100000000ULL) }; + const v_type cdef{ static_cast(cdefghij * 3518437209ULL >> 45) }; + pow2MemcpyWrapper<2ULL>(buf, charTable2DigitData + ab); + pow2MemcpyWrapper<4ULL>(buf + 2, charTable4DigitData + cdef); + pow2MemcpyWrapper<4ULL>(buf + 6, charTable4DigitData + cdefghij - cdef * 10000ULL); + return buf + 10; + } + }; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type abc{ multiply_and_shift::impl(value) }; + const v_type defghijk{ value - abc * 100000000ULL }; + const v_type defg{ defghijk * 3518437209ULL >> 45 }; + copy3Digits(buf, abc); + pow2MemcpyWrapper<4ULL>(buf + 3, charTable4DigitData + defg); + pow2MemcpyWrapper<4ULL>(buf + 7, charTable4DigitData + defghijk - defg * 10000ULL); + return buf + 11; + } + }; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type abcd{ multiply_and_shift::impl(value) }; + const v_type efghijkl{ value - abcd * 100000000ULL }; + const v_type efgh{ efghijkl * 3518437209ULL >> 45 }; + pow2MemcpyWrapper<4ULL>(buf, charTable4DigitData + abcd); + pow2MemcpyWrapper<4ULL>(buf + 4, charTable4DigitData + efgh); + pow2MemcpyWrapper<4ULL>(buf + 8, charTable4DigitData + efghijkl - efgh * 10000ULL); + return buf + 12; + } + }; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type abcde{ multiply_and_shift::impl(value) }; + const v_type fghijklm{ value - abcde * 100000000ULL }; + const v_type a{ abcde * 3518437209ULL >> 45 }; + const v_type fghi{ fghijklm * 3518437209ULL >> 45 }; + *buf = static_cast(a + '0'); + pow2MemcpyWrapper<4ULL>(buf + 1, charTable4DigitData + abcde - a * 10000ULL); + pow2MemcpyWrapper<4ULL>(buf + 5, charTable4DigitData + fghi); + pow2MemcpyWrapper<4ULL>(buf + 9, charTable4DigitData + fghijklm - fghi * 10000ULL); + return buf + 13; + } + }; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type abcdef{ multiply_and_shift::impl(value) }; + const v_type ghijklmn{ value - abcdef * 100000000ULL }; + const v_type ab{ abcdef * 3518437209ULL >> 45 }; + const v_type ghij{ ghijklmn * 3518437209ULL >> 45 }; + pow2MemcpyWrapper<2ULL>(buf, charTable2DigitData + ab); + pow2MemcpyWrapper<4ULL>(buf + 2, charTable4DigitData + abcdef - ab * 10000ULL); + pow2MemcpyWrapper<4ULL>(buf + 6, charTable4DigitData + ghij); + pow2MemcpyWrapper<4ULL>(buf + 10, charTable4DigitData + ghijklmn - ghij * 10000ULL); + return buf + 14; + } + }; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type abcdefg{ multiply_and_shift::impl(value) }; + const v_type hijklmno{ value - abcdefg * 100000000ULL }; + const v_type abc{ abcdefg * 3518437209ULL >> 45 }; + const v_type hijk{ hijklmno * 3518437209ULL >> 45 }; + copy3Digits(buf, abc); + pow2MemcpyWrapper<4ULL>(buf + 3, charTable4DigitData + abcdefg - abc * 10000ULL); + pow2MemcpyWrapper<4ULL>(buf + 7, charTable4DigitData + hijk); + pow2MemcpyWrapper<4ULL>(buf + 11, charTable4DigitData + hijklmno - hijk * 10000ULL); + return buf + 15; + } + }; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type abcdefgh{ multiply_and_shift::impl(value) }; + const v_type ijklmnop{ value - abcdefgh * 100000000ULL }; + const v_type abcd{ abcdefgh * 3518437209ULL >> 45 }; + const v_type ijkl{ ijklmnop * 3518437209ULL >> 45 }; + pow2MemcpyWrapper<4ULL>(buf, charTable4DigitData + abcd); + pow2MemcpyWrapper<4ULL>(buf + 4, charTable4DigitData + abcdefgh - abcd * 10000ULL); + pow2MemcpyWrapper<4ULL>(buf + 8, charTable4DigitData + ijkl); + pow2MemcpyWrapper<4ULL>(buf + 12, charTable4DigitData + ijklmnop - ijkl * 10000ULL); + return buf + 16; + } + }; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type abcdefghi{ multiply_and_shift::impl(value) }; + const v_type jklmnopq{ value - abcdefghi * 100000000ULL }; + const v_type a{ multiply_and_shift::impl(abcdefghi) }; + const v_type bcdefghi{ abcdefghi - a * 100000000ULL }; + const v_type bcde{ bcdefghi * 3518437209ULL >> 45 }; + const v_type jklm{ jklmnopq * 3518437209ULL >> 45 }; + *buf = static_cast(a + '0'); + pow2MemcpyWrapper<4ULL>(buf + 1, charTable4DigitData + bcde); + pow2MemcpyWrapper<4ULL>(buf + 5, charTable4DigitData + bcdefghi - bcde * 10000ULL); + pow2MemcpyWrapper<4ULL>(buf + 9, charTable4DigitData + jklm); + pow2MemcpyWrapper<4ULL>(buf + 13, charTable4DigitData + jklmnopq - jklm * 10000ULL); + return buf + 17; + } + }; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type abcdefghij{ multiply_and_shift::impl(value) }; + const v_type klmnopqr{ value - abcdefghij * 100000000ULL }; + const v_type ab{ multiply_and_shift::impl(abcdefghij) }; + const v_type cdefghij{ abcdefghij - ab * 100000000ULL }; + const v_type cdef{ cdefghij * 3518437209ULL >> 45 }; + const v_type klmn{ klmnopqr * 3518437209ULL >> 45 }; + pow2MemcpyWrapper<2ULL>(buf, charTable2DigitData + ab); + pow2MemcpyWrapper<4ULL>(buf + 2, charTable4DigitData + cdef); + pow2MemcpyWrapper<4ULL>(buf + 6, charTable4DigitData + cdefghij - cdef * 10000ULL); + pow2MemcpyWrapper<4ULL>(buf + 10, charTable4DigitData + klmn); + pow2MemcpyWrapper<4ULL>(buf + 14, charTable4DigitData + klmnopqr - klmn * 10000ULL); + return buf + 18; + } + }; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type abcdefghijk{ multiply_and_shift::impl(value) }; + const v_type lmnopqrs{ value - abcdefghijk * 100000000ULL }; + const v_type abc{ multiply_and_shift::impl(abcdefghijk) }; + const v_type defghijk{ abcdefghijk - abc * 100000000ULL }; + const v_type defg{ defghijk * 3518437209ULL >> 45 }; + const v_type lmno{ lmnopqrs * 3518437209ULL >> 45 }; + copy3Digits(buf, abc); + pow2MemcpyWrapper<4ULL>(buf + 3, charTable4DigitData + defg); + pow2MemcpyWrapper<4ULL>(buf + 7, charTable4DigitData + defghijk - defg * 10000ULL); + pow2MemcpyWrapper<4ULL>(buf + 11, charTable4DigitData + lmno); + pow2MemcpyWrapper<4ULL>(buf + 15, charTable4DigitData + lmnopqrs - lmno * 10000ULL); + return buf + 19; + } + }; + + template struct to_chars_internal { + inline static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + const v_type abcdefghijkl{ multiply_and_shift::impl(value) }; + const v_type mnopqrst{ value - abcdefghijkl * 100000000ULL }; + const v_type abcd{ multiply_and_shift::impl(abcdefghijkl) }; + const v_type efghijkl{ abcdefghijkl - abcd * 100000000ULL }; + const v_type efgh{ efghijkl * 3518437209ULL >> 45 }; + const v_type mnop{ mnopqrst * 3518437209ULL >> 45 }; + pow2MemcpyWrapper<4ULL>(buf, charTable4DigitData + abcd); + pow2MemcpyWrapper<4ULL>(buf + 4, charTable4DigitData + efgh); + pow2MemcpyWrapper<4ULL>(buf + 8, charTable4DigitData + efghijkl - efgh * 10000ULL); + pow2MemcpyWrapper<4ULL>(buf + 12, charTable4DigitData + mnop); + pow2MemcpyWrapper<4ULL>(buf + 16, charTable4DigitData + mnopqrst - mnop * 10000ULL); + return buf + 20; + } + }; + + template + JSONIFIER_INLINE unrestrict_t implInternal(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + return to_chars_internal::impl(buf, value); + } + + template struct to_chars { + JSONIFIER_INLINE static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + return value < 100000000ULL ? value < 10000ULL ? value < 100ULL ? value < 10U ? (static_cast(buf[0] = static_cast(value + '0')), buf + 1) + : (static_cast(pow2MemcpyWrapper<2ULL>(buf, charTable2DigitData + value)), buf + 2) + : value < 1000U ? (static_cast(copy3Digits(buf, value)), buf + 3) + : (static_cast(pow2MemcpyWrapper<4ULL>(buf, charTable4DigitData + value)), buf + 4) + : value < 1000000ULL ? value < 100000ULL ? implInternal<5ULL>(buf, value) : implInternal<6ULL>(buf, value) + : value < 10000000ULL ? implInternal<7ULL>(buf, value) + : implInternal<8ULL>(buf, value) + : value < 1000000000000ULL ? value < 10000000000ULL ? value < 1000000000ULL ? implInternal<9ULL>(buf, value) : implInternal<10ULL>(buf, value) + : value < 100000000000ULL ? implInternal<11ULL>(buf, value) + : implInternal<12ULL>(buf, value) + : value < 10000000000000000ULL ? value < 100000000000000ULL ? value < 10000000000000ULL ? implInternal<13ULL>(buf, value) : implInternal<14ULL>(buf, value) + : value < 1000000000000000ULL ? implInternal<15ULL>(buf, value) + : implInternal<16ULL>(buf, value) + : value < 1000000000000000000ULL ? value < 100000000000000000ULL ? implInternal<17ULL>(buf, value) : implInternal<18ULL>(buf, value) + : value < 10000000000000000000ULL ? implInternal<19ULL>(buf, value) + : implInternal<20ULL>(buf, value); + } + }; + + template struct to_chars { + JSONIFIER_INLINE static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + return value < 100000U ? value < 1000U ? value < 100U ? value < 10U ? (static_cast(buf[0] = static_cast(value + '0')), buf + 1) + : (static_cast(pow2MemcpyWrapper<2ULL>(buf, charTable2DigitData + value)), buf + 2) + : (static_cast(copy3Digits(buf, value)), buf + 3) + : value < 10000ULL ? (static_cast(pow2MemcpyWrapper<4ULL>(buf, charTable4DigitData + value)), buf + 4) + : implInternal<5ULL>(buf, value) + : value < 10000000U ? value < 1000000U ? implInternal<6ULL>(buf, value) : implInternal<7ULL>(buf, value) + : value < 1000000000U ? value < 100000000U ? implInternal<8ULL>(buf, value) : implInternal<9ULL>(buf, value) + : implInternal<10ULL>(buf, value); + } + }; + + template struct to_chars { + JSONIFIER_INLINE static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + return value < 1000U ? value < 100U ? value < 10U ? (static_cast(buf[0] = static_cast(value + '0')), buf + 1) + : (static_cast(pow2MemcpyWrapper<2ULL>(buf, charTable2DigitData + value)), buf + 2) + : (static_cast(copy3Digits(buf, value)), buf + 3) + : value < 10000ULL ? (static_cast(pow2MemcpyWrapper<4ULL>(buf, charTable4DigitData + value)), buf + 4) + : implInternal<5ULL>(buf, value); + } + }; + + template struct to_chars { + JSONIFIER_INLINE static unrestrict_t impl(write_buffer_ptr buf, const v_type value) noexcept { + return value < 100 ? value < 10 ? (static_cast(buf[0] = static_cast(value + '0')), buf + 1) + : (static_cast(pow2MemcpyWrapper<2>(buf, &charTable2DigitData[value])), buf + 2) + : (static_cast(copy3Digits(buf, value)), buf + 3); + } + }; + + template struct to_chars { + JSONIFIER_INLINE static unrestrict_t implNegative(write_buffer_ptr buf, const v_type value) noexcept { + using unsigned_type = std::make_unsigned_t; + static constexpr unsigned_type shiftAmount = static_cast(sizeof(v_type) * 8ULL - 1ULL); + *buf = '-'; + return to_chars::impl(buf + 1, + static_cast( + (static_cast(value) ^ static_cast(value >> shiftAmount)) - static_cast(value >> shiftAmount))); + } + + JSONIFIER_INLINE static unrestrict_t impl(write_buffer_ptr buf JSONIFIER_LIFETIME_BOUND, const v_type value) noexcept { + using unsigned_type = std::make_unsigned_t; + static constexpr unsigned_type shiftAmount = static_cast(sizeof(v_type) * 8ULL - 1ULL); + return (value < 0) ? implNegative(buf, value) + : to_chars::impl(buf, + static_cast((static_cast(value) ^ static_cast(value >> shiftAmount)) - + static_cast(value >> shiftAmount))); + } + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/simd.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/simd.hpp new file mode 100644 index 000000000000..5e573c4d23fe --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/simd.hpp @@ -0,0 +1,858 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/simd.hpp + */ +// Sampled from Dr. Lemire's library, simdjson: https://github.com/simdjson/simdjson +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace jsonifier::internal { + + inline void printBitsAligned(uint64_t bits, const char* label, read_buffer_ptr str = nullptr, uint64_t len = 0) noexcept { + cout << label << ":" << endl; + if (str && len > 0) { + cout << "STR: "; + for (uint64_t i = 0; i < std::min(len, 64); ++i) { + char c = static_cast(str[i]); + if (c == '\n' || c == '\r' || c == '\t') { + c = ' '; + } + cout << c; + } + cout << endl; + } + cout << "BITS: "; + for (uint64_t i = 0; i < 64; ++i) { + cout << ((bits >> i) & 1ULL); + } + cout << endl; + cout << "IDX: "; + for (uint64_t i = 0; i < 64; ++i) { + cout << (i % 10); + } + cout << endl; + cout << "TENS: "; + for (uint64_t i = 0; i < 64; ++i) { + cout << ((i / 10) % 10); + } + cout << endl << endl; + } + + struct tail_block_builder { + static constexpr uint64_t pageSize = 4096; + + alignas(64) static constexpr uint8_t laneIndices[64]{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 }; + + JSONIFIER_INLINE static void buildInPlace(write_buffer_ptr dst, read_buffer_ptr src, uint64_t remaining, uint64_t paddedBytes) noexcept { + const jsonifier_simd_int_t spaces = simd::gatherValue(static_cast(0x20)); + const jsonifier_simd_int_t lanes = simd::gatherValues(+laneIndices); + for (uint64_t offset = 0; offset < paddedBytes; offset += simdBytesPerRegister) { + const uint64_t lanesLeft = remaining > offset ? std::min(remaining - offset, simdBytesPerRegister) : 0; + const jsonifier_simd_int_t keep = simd::opCmpLtRaw(lanes, simd::gatherValue(static_cast(lanesLeft))); + const jsonifier_simd_int_t data = simd::gatherValuesU(src + offset); + simd::storeU(simd::opXor(spaces, simd::opAnd(simd::opXor(data, spaces), keep)), dst + offset); + } + } + + JSONIFIER_INLINE static void buildByCopy(write_buffer_ptr dst, read_buffer_ptr src, uint64_t remaining, uint64_t paddedBytes) noexcept { + std::memset(dst + remaining, static_cast(0x20), paddedBytes - remaining); + jsonifierMemcpy(dst, src, remaining); + } + + JSONIFIER_INLINE static void impl(write_buffer_ptr dst, read_buffer_ptr src, uint64_t remaining, uint64_t paddedBytes) noexcept { + if (readStaysInPage(src, remaining, paddedBytes)) [[likely]] { + buildInPlace(dst, src, remaining, paddedBytes); + } else { + buildByCopy(dst, src, remaining, paddedBytes); + } + } + + JSONIFIER_INLINE static bool readStaysInPage(read_buffer_ptr src, uint64_t remaining, uint64_t paddedBytes) noexcept { + if constexpr (JSONIFIER_ASAN_ENABLED) { + return false; + } else { + const uint64_t lastValid = static_cast(std::bit_cast(src)) + remaining - 1; + const uint64_t lastRead = static_cast(std::bit_cast(src)) + paddedBytes - 1; + return (lastValid ^ lastRead) < pageSize; + } + } + }; + + struct string_block_reader { + static constexpr uint64_t stepBytes = simdBlocksPerStep * 64; + + JSONIFIER_INLINE unrestrict_t getRemainder(uint64_t paddedBytes) noexcept { + tail_block_builder::impl(+block, inString + index, length - index, paddedBytes); + return +block; + } + + JSONIFIER_INLINE void reset(read_buffer_ptr stringViewNew, uint64_t lengthNew) noexcept { + lengthMinusStep = lengthNew < stepBytes ? 0 : lengthNew - stepBytes; + inString = std::bit_cast(stringViewNew); + length = lengthNew; + index = 0; + } + + JSONIFIER_INLINE unrestrict_t fullBlock() noexcept { + read_buffer_ptr newPtr = inString + index; + index += stepBytes; + return newPtr; + } + + JSONIFIER_INLINE uint64_t getRemainderBytes() const noexcept { + return length - index; + } + + JSONIFIER_INLINE bool hasFullBlock() const noexcept { + return index < lengthMinusStep; + } + + alignas(simdBytesPerRegister) uint8_t block[stepBytes]{}; + uint64_t lengthMinusStep{}; + read_buffer_ptr inString{}; + uint64_t length{}; + uint64_t index{}; + }; + + struct string_block_reader_locals { + static constexpr uint64_t stepBytes = simdBlocksPerStep * 64; + + JSONIFIER_INLINE unrestrict_t getRemainder(uint64_t paddedBytes) noexcept { + tail_block_builder::impl(+block, inString + index, length - index, paddedBytes); + return +block; + } + + JSONIFIER_INLINE void reset(read_buffer_ptr stringViewNew, uint64_t lengthNew) noexcept { + lengthMinusStep = lengthNew < stepBytes ? 0 : lengthNew - stepBytes; + inString = std::bit_cast(stringViewNew); + length = lengthNew; + index = 0; + } + + alignas(simdBytesPerRegister) uint8_t block[stepBytes]{}; + read_buffer_ptr inString{}; + uint64_t lengthMinusStep{}; + uint64_t length{}; + uint64_t index{}; + }; + + template struct pod_block_reader { + JSONIFIER_INLINE unrestrict_t getRemainder() noexcept { + const uint64_t remaining = lengthVal - indexVal; + read_buffer_ptr srcPtr = inString + indexVal; + if (tail_block_builder::readStaysInPage(srcPtr, remaining, simdBytesPerBlock)) [[likely]] { + tail_block_builder::buildInPlace(+block, srcPtr, remaining, simdBytesPerBlock); + } else { + jsonifierMemcpyUpTo(+block, srcPtr, remaining); + std::memset(+block + remaining, static_cast(0x20), simdBytesPerBlock - remaining); + } + return +block; + } + + JSONIFIER_INLINE void reset(read_buffer_ptr stringViewNew, uint64_t lengthNew) noexcept { + fullStepEnd = lengthNew & ~(stepBytes - 1ull); + fullBlockEnd = lengthNew & ~(simdBytesPerBlock - 1ull); + inString = std::bit_cast(stringViewNew); + lengthVal = lengthNew; + indexVal = 0; + } + + JSONIFIER_INLINE unrestrict_t fullBlock() noexcept { + read_buffer_ptr newPtr = inString + indexVal; + indexVal += simdBytesPerBlock; + return newPtr; + } + + JSONIFIER_INLINE unrestrict_t fullStep() noexcept { + read_buffer_ptr newPtr = inString + indexVal; + indexVal += stepBytes; + return newPtr; + } + + JSONIFIER_INLINE uint64_t getRemainderBytes() const noexcept { + return lengthVal - indexVal; + } + + JSONIFIER_INLINE bool hasFullBlock() const noexcept { + return indexVal < fullBlockEnd; + } + + JSONIFIER_INLINE bool hasFullStep() const noexcept { + return indexVal < fullStepEnd; + } + + JSONIFIER_INLINE uint64_t length() const noexcept { + return lengthVal; + } + + JSONIFIER_INLINE uint64_t index() const noexcept { + return indexVal; + } + + alignas(simdBytesPerRegister) uint8_t block[simdBytesPerBlock]; + read_buffer_ptr inString{}; + uint64_t fullBlockEnd{}; + uint64_t fullStepEnd{}; + uint64_t lengthVal{}; + uint64_t indexVal{}; + + static_assert((stepBytes & (stepBytes - 1)) == 0 && stepBytes >= simdBytesPerBlock); + }; + + struct rope_block { + JSONIFIER_INLINE uint64_t nonQuoteOutsideString(uint64_t mask) const noexcept { + return mask & ~inString; + } + + JSONIFIER_INLINE uint64_t stringTail() const noexcept { + return inString ^ quotes; + } + + uint64_t inString{}; + uint64_t escaped{}; + uint64_t quotes{}; + }; + + struct simd_string_reader_members : simd::rope_detector, string_block_reader, add_tape_values>, alloc_wrapper { + friend add_tape_values>; + using allocator = alloc_wrapper; + + JSONIFIER_INLINE simd_string_reader_members() noexcept = default; + + JSONIFIER_INLINE simd_string_reader_members& operator=(simd_string_reader_members&& other) noexcept { + if (&other != this) { + swap(tapeCount, other.tapeCount); + swap(capacity, other.capacity); + swap(string_block_reader::length, other.string_block_reader::length); + swap(tape, other.tape); + } + return *this; + } + + JSONIFIER_INLINE simd_string_reader_members(simd_string_reader_members&& other) noexcept : allocator{} { + *this = internal::move(other); + } + + simd_string_reader_members& operator=(const simd_string_reader_members& other) = delete; + simd_string_reader_members(const simd_string_reader_members& other) = delete; + + template JSONIFIER_INLINE void reset(read_buffer_ptr rootIter, uint64_t stringLength) noexcept { + const uint64_t neededCapacity = stringLength + 64; + if (neededCapacity > capacity) { + auto newTape = allocator::allocate(neededCapacity); + if (tape) { + allocator::deallocate(tape, capacity); + } + tape = newTape; + capacity = neededCapacity; + } + + tapeCount = 0; + string_block_reader::reset(rootIter, stringLength); + simd::rope_detector::prevInString = 0; + simd::rope_detector::prevScalar = 0; + simd::rope_detector::nextIsEscaped = 0; + + const jsonifier_simd_int_t bsRegister = simd::gatherValue('\\'); + const jsonifier_simd_int_t quoteRegister = simd::gatherValue('"'); + const jsonifier_simd_int_t opTable = simd::gatherValues(simd::opArray.data()); + const jsonifier_simd_int_t spaceMask = simd::gatherValue(static_cast(0x20)); + + if constexpr (minified) { + resetImpl(bsRegister, quoteRegister, opTable, spaceMask); + } else { + const jsonifier_simd_int_t whitespaceTableLocal = simd::gatherValues(simd::whitespaceArray.data()); + resetImpl(bsRegister, quoteRegister, opTable, spaceMask, whitespaceTableLocal); + } + } + + JSONIFIER_INLINE unrestrict_t begin() noexcept { + tape[tapeCount] = static_cast(string_block_reader::length); + return tape; + } + + JSONIFIER_INLINE unrestrict_t end() noexcept { + return tape + tapeCount; + } + + JSONIFIER_INLINE uint64_t getTapeCount() noexcept { + return tapeCount; + } + + JSONIFIER_INLINE ~simd_string_reader_members() noexcept { + if (tape) { + allocator::deallocate(tape, capacity); + tape = nullptr; + } + } + + protected: + template JSONIFIER_INLINE void processBlocks(read_buffer_ptr blockPtr, uint64_t stepBaseIndex, const jsonifier_simd_int_t bsRegister, + const jsonifier_simd_int_t quoteRegister, const jsonifier_simd_int_t opTable, const jsonifier_simd_int_t spaceMask, const jsonifier_simd_int_types... args) noexcept { + array bitsArr; + array cntsArr; + processBlocksImpl<0>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + if constexpr (simdBlocksPerStep > 1) { + processBlocksImpl<1>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + if constexpr (simdBlocksPerStep > 2) { + processBlocksImpl<2>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + processBlocksImpl<3>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + if constexpr (simdBlocksPerStep > 4) { + processBlocksImpl<4>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + processBlocksImpl<5>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + processBlocksImpl<6>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + processBlocksImpl<7>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + } + } + } + + add_tape_values>::impl(bitsArr, cntsArr, tape + tapeCount, stepBaseIndex); + + tapeCount += cntsArr[0]; + if constexpr (simdBlocksPerStep > 1) { + tapeCount += cntsArr[1]; + if constexpr (simdBlocksPerStep > 2) { + tapeCount += cntsArr[2]; + tapeCount += cntsArr[3]; + if constexpr (simdBlocksPerStep > 4) { + tapeCount += cntsArr[4]; + tapeCount += cntsArr[5]; + tapeCount += cntsArr[6]; + tapeCount += cntsArr[7]; + } + } + } + } + + template JSONIFIER_INLINE void resetImpl(const jsonifier_simd_int_t bsRegister, + const jsonifier_simd_int_t quoteRegister, const jsonifier_simd_int_t opTable, const jsonifier_simd_int_t spaceMask, const jsonifier_simd_int_types... args) noexcept { + while (string_block_reader::hasFullBlock()) { + const uint64_t stepBaseIndex = string_block_reader::index; + processBlocks(string_block_reader::fullBlock(), stepBaseIndex, bsRegister, quoteRegister, opTable, spaceMask, args...); + } + + if (const uint64_t remaining = string_block_reader::getRemainderBytes(); remaining != 0) { + const uint64_t tailBlocks = (remaining + 63) / 64; + read_buffer_ptr tailPtr = string_block_reader::getRemainder(tailBlocks * 64); + for (uint64_t blockIndex = 0; blockIndex < tailBlocks; ++blockIndex) { + processTailBlock(tailPtr + blockIndex * 64, string_block_reader::index + blockIndex * 64, bsRegister, quoteRegister, opTable, spaceMask, args...); + } + while (tapeCount > 0 && tape[tapeCount - 1] >= string_block_reader::length) { + --tapeCount; + } + } + } + + template JSONIFIER_INLINE void processBlocksImpl(array& bitsArr, + array& cntsArr, read_buffer_ptr blockPtr, const jsonifier_simd_int_t bsRegister, const jsonifier_simd_int_t quoteRegister, + const jsonifier_simd_int_t opTable, const jsonifier_simd_int_t spaceMask, const jsonifier_simd_int_types... args) noexcept { + simd_array_t inVals; + inVals.template set<0>(simd::gatherValuesU(blockPtr + currentBlock * 64)); + if constexpr (simdRegistersPerBlock > 1) { + inVals.template set<1>(simd::gatherValuesU(blockPtr + currentBlock * 64 + simdBytesPerRegister * 1)); + if constexpr (simdRegistersPerBlock > 2) { + inVals.template set<2>(simd::gatherValuesU(blockPtr + currentBlock * 64 + simdBytesPerRegister * 2)); + inVals.template set<3>(simd::gatherValuesU(blockPtr + currentBlock * 64 + simdBytesPerRegister * 3)); + } + } + simd::rope_detector::next(inVals, bsRegister, quoteRegister); + const uint64_t structurals = getStructurals(inVals, opTable, spaceMask, args...) & ~simd::rope_detector::stringTail(); + bitsArr[currentBlock] = structurals; + cntsArr[currentBlock] = simd::tape_writer_op::correctedPopcount(structurals); + } + + template JSONIFIER_INLINE void processTailBlock(read_buffer_ptr blockPtr, uint64_t blockBaseIndex, + const jsonifier_simd_int_t bsRegister, const jsonifier_simd_int_t quoteRegister, const jsonifier_simd_int_t opTable, const jsonifier_simd_int_t spaceMask, + const jsonifier_simd_int_types... args) noexcept { + array bitsArr; + array cntsArr; + processBlocksImpl<0>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + add_tape_values>::impl(bitsArr, cntsArr, tape + tapeCount, blockBaseIndex); + tapeCount += cntsArr[0]; + } + + JSONIFIER_INLINE uint64_t getStructurals(const simd_array_t in01, const jsonifier_simd_int_t opTable, const jsonifier_simd_int_t spaceMask) noexcept { + const uint64_t op = simd::op_collector::impl(in01, opTable, spaceMask); + const uint64_t scalar = ~(op | simd::rope_detector::quotes); + const uint64_t follows = simd::rope_detector::followsNonquoteScalar(scalar); + const uint64_t scalarStart = scalar & ~follows; + return op | simd::rope_detector::quotes | scalarStart; + } + + JSONIFIER_INLINE uint64_t getStructurals(const simd_array_t in01, const jsonifier_simd_int_t opTable, const jsonifier_simd_int_t spaceMask, + const jsonifier_simd_int_t whitespaceTableLocal) noexcept { +#if JSONIFIER_NEON + const uint64x2_t wsOp = simd::ws_op_collector::impl(in01, opTable, spaceMask, whitespaceTableLocal); + const uint64_t whitespace = vgetq_lane_u64(wsOp, 0); + const uint64_t op = vgetq_lane_u64(wsOp, 1); +#else + const uint64_t whitespace = simd::ws_collector::impl(in01, whitespaceTableLocal); + const uint64_t op = simd::op_collector::impl(in01, opTable, spaceMask); +#endif + const uint64_t scalar = ~(op | whitespace | simd::rope_detector::quotes); + const uint64_t follows = simd::rope_detector::followsNonquoteScalar(scalar); + const uint64_t scalarStart = scalar & ~follows; + return op | simd::rope_detector::quotes | scalarStart; + } + + write_structural_index_ptr tape{}; + uint64_t tapeCount{}; + uint64_t capacity{}; + }; + + struct simd_string_reader_locals : string_block_reader_locals, add_tape_values>, alloc_wrapper { + friend add_tape_values>; + using allocator = alloc_wrapper; + using rope_type = simd::rope_detector; + + JSONIFIER_INLINE simd_string_reader_locals() noexcept = default; + + JSONIFIER_INLINE simd_string_reader_locals& operator=(simd_string_reader_locals&& other) noexcept { + if (&other != this) { + swap(tapeCount, other.tapeCount); + swap(capacity, other.capacity); + swap(string_block_reader_locals::length, other.string_block_reader_locals::length); + swap(tape, other.tape); + } + return *this; + } + + JSONIFIER_INLINE simd_string_reader_locals(simd_string_reader_locals&& other) noexcept : allocator{} { + *this = internal::move(other); + } + + simd_string_reader_locals& operator=(const simd_string_reader_locals& other) = delete; + simd_string_reader_locals(const simd_string_reader_locals& other) = delete; + +#if JSONIFIER_COMPILER_CLANG && JSONIFIER_NEON + template JSONIFIER_INLINE static void collectCountsQuad(const array& __restrict bitsArr, + array& __restrict cntsArr) noexcept { + const uint8x16_t counts01 = vcntq_u8(vreinterpretq_u8_u64(vcombine_u64(vcreate_u64(bitsArr[offset]), vcreate_u64(bitsArr[offset + 1])))); + const uint8x16_t counts23 = vcntq_u8(vreinterpretq_u8_u64(vcombine_u64(vcreate_u64(bitsArr[offset + 2]), vcreate_u64(bitsArr[offset + 3])))); + uint8x16_t sums = vpaddq_u8(counts01, counts23); + sums = vpaddq_u8(sums, sums); + sums = vpaddq_u8(sums, sums); + const uint32_t packed = vgetq_lane_u32(vreinterpretq_u32_u8(sums), 0); + cntsArr[offset] = packed & 0xFFu; + cntsArr[offset + 1] = (packed >> 8) & 0xFFu; + cntsArr[offset + 2] = (packed >> 16) & 0xFFu; + cntsArr[offset + 3] = packed >> 24; + } + + JSONIFIER_INLINE static void collectCounts(const array& __restrict bitsArr, array& __restrict cntsArr) noexcept { + collectCountsQuad<0>(bitsArr, cntsArr); + if constexpr (simdBlocksPerStep > 4) { + collectCountsQuad<4>(bitsArr, cntsArr); + } + } + + static_assert(simdBlocksPerStep == 4 || simdBlocksPerStep == 8); +#endif + + template JSONIFIER_INLINE static void processBlocksImpl(rope_type& __restrict rope, + array& __restrict bitsArr, array& __restrict cntsArr, read_buffer_ptr blockPtr, const jsonifier_simd_int_t bsRegister, + const jsonifier_simd_int_t quoteRegister, const jsonifier_simd_int_t opTable, const jsonifier_simd_int_t spaceMask, const jsonifier_simd_int_types... args) noexcept { + simd_array_t inVals; + inVals.template set<0>(simd::gatherValuesU(blockPtr + currentBlock * 64)); + if constexpr (simdRegistersPerBlock > 1) { + inVals.template set<1>(simd::gatherValuesU(blockPtr + currentBlock * 64 + simdBytesPerRegister * 1)); + if constexpr (simdRegistersPerBlock > 2) { + inVals.template set<2>(simd::gatherValuesU(blockPtr + currentBlock * 64 + simdBytesPerRegister * 2)); + inVals.template set<3>(simd::gatherValuesU(blockPtr + currentBlock * 64 + simdBytesPerRegister * 3)); + } + } + rope.next(inVals, bsRegister, quoteRegister); + const uint64_t structurals = getStructurals(rope, inVals, opTable, spaceMask, args...) & ~rope.stringTail(); + bitsArr[currentBlock] = structurals; +#if JSONIFIER_COMPILER_CLANG && JSONIFIER_NEON + if constexpr (laneCount == 1) { + cntsArr[currentBlock] = simd::tape_writer_op::correctedPopcount(structurals); + } +#else + cntsArr[currentBlock] = simd::tape_writer_op::correctedPopcount(structurals); +#endif + } + + template JSONIFIER_INLINE void resetImpl(const jsonifier_simd_int_t bsRegister, + const jsonifier_simd_int_t quoteRegister, const jsonifier_simd_int_t opTable, const jsonifier_simd_int_t spaceMask, const jsonifier_simd_int_types... args) noexcept { + rope_type rope{}; + write_structural_index_ptr tapeLocal = tape; + read_buffer_ptr srcPtr = string_block_reader_locals::inString; + const uint64_t lengthMinusStepLocal = string_block_reader_locals::lengthMinusStep; + const uint64_t lengthLocal = string_block_reader_locals::length; + uint64_t tapeCountLocal = 0; + uint64_t indexLocal = 0; + + while (indexLocal < lengthMinusStepLocal) { + tapeCountLocal += processBlocks(rope, srcPtr + indexLocal, tapeLocal + tapeCountLocal, indexLocal, bsRegister, quoteRegister, opTable, spaceMask, args...); + indexLocal += stepBytes; + } + string_block_reader_locals::index = indexLocal; + + if (const uint64_t remaining = lengthLocal - indexLocal; remaining != 0) { + const uint64_t tailBlocks = (remaining + 63) / 64; + read_buffer_ptr tailPtr = string_block_reader_locals::getRemainder(tailBlocks * 64); + for (uint64_t blockIndex = 0; blockIndex < tailBlocks; ++blockIndex) { + tapeCountLocal += processTailBlock(rope, tailPtr + blockIndex * 64, tapeLocal + tapeCountLocal, indexLocal + blockIndex * 64, bsRegister, quoteRegister, + opTable, spaceMask, args...); + } + while (tapeCountLocal > 0 && tapeLocal[tapeCountLocal - 1] >= lengthLocal) { + --tapeCountLocal; + } + } + tapeCount = tapeCountLocal; + } + + template JSONIFIER_INLINE static uint64_t processTailBlock(rope_type& __restrict rope, read_buffer_ptr blockPtr, + write_structural_index_ptr tapePtr, uint64_t blockBaseIndex, const jsonifier_simd_int_t bsRegister, const jsonifier_simd_int_t quoteRegister, + const jsonifier_simd_int_t opTable, const jsonifier_simd_int_t spaceMask, const jsonifier_simd_int_types... args) noexcept { + array bitsArr; + array cntsArr; + processBlocksImpl<0>(rope, bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + add_tape_values>::impl(bitsArr, cntsArr, tapePtr, blockBaseIndex); + return cntsArr[0]; + } + + template JSONIFIER_INLINE static uint64_t processBlocks(rope_type& __restrict rope, read_buffer_ptr blockPtr, + write_structural_index_ptr tapePtr, uint64_t stepBaseIndex, const jsonifier_simd_int_t bsRegister, const jsonifier_simd_int_t quoteRegister, + const jsonifier_simd_int_t opTable, const jsonifier_simd_int_t spaceMask, const jsonifier_simd_int_types... args) noexcept { + array bitsArr; + array cntsArr; + processBlocksImpl<0>(rope, bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + if constexpr (simdBlocksPerStep > 1) { + processBlocksImpl<1>(rope, bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + if constexpr (simdBlocksPerStep > 2) { + processBlocksImpl<2>(rope, bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + processBlocksImpl<3>(rope, bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + if constexpr (simdBlocksPerStep > 4) { + processBlocksImpl<4>(rope, bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + processBlocksImpl<5>(rope, bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + processBlocksImpl<6>(rope, bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + processBlocksImpl<7>(rope, bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + } + } + } +#if JSONIFIER_COMPILER_CLANG && JSONIFIER_NEON + collectCounts(bitsArr, cntsArr); +#endif + + add_tape_values>::impl(bitsArr, cntsArr, tapePtr, stepBaseIndex); + + uint64_t stepCount = cntsArr[0]; + if constexpr (simdBlocksPerStep > 1) { + stepCount += cntsArr[1]; + if constexpr (simdBlocksPerStep > 2) { + stepCount += cntsArr[2]; + stepCount += cntsArr[3]; + if constexpr (simdBlocksPerStep > 4) { + stepCount += cntsArr[4]; + stepCount += cntsArr[5]; + stepCount += cntsArr[6]; + stepCount += cntsArr[7]; + } + } + } + return stepCount; + } + + template JSONIFIER_INLINE void reset(read_buffer_ptr rootIter, uint64_t stringLength) noexcept { + const uint64_t neededCapacity = stringLength + 64; + if (neededCapacity > capacity) { + auto newTape = allocator::allocate(neededCapacity); + if (tape) { + allocator::deallocate(tape, capacity); + } + tape = newTape; + capacity = neededCapacity; + } + + string_block_reader_locals::reset(rootIter, stringLength); + + const jsonifier_simd_int_t bsRegister = simd::gatherValue('\\'); + const jsonifier_simd_int_t quoteRegister = simd::gatherValue('"'); + const jsonifier_simd_int_t opTable = simd::gatherValues(simd::opArray.data()); + const jsonifier_simd_int_t spaceMask = simd::gatherValue(static_cast(0x20)); + + if constexpr (minified) { + resetImpl(bsRegister, quoteRegister, opTable, spaceMask); + } else { + const jsonifier_simd_int_t whitespaceTableLocal = simd::gatherValues(simd::whitespaceArray.data()); + resetImpl(bsRegister, quoteRegister, opTable, spaceMask, whitespaceTableLocal); + } + } + + JSONIFIER_INLINE static uint64_t getStructurals(rope_type& __restrict rope, const simd_array_t in01, const jsonifier_simd_int_t opTable, + const jsonifier_simd_int_t spaceMask, const jsonifier_simd_int_t whitespaceTableLocal) noexcept { +#if JSONIFIER_NEON + const uint64x2_t wsOp = simd::ws_op_collector::impl(in01, opTable, spaceMask, whitespaceTableLocal); + const uint64_t whitespace = vgetq_lane_u64(wsOp, 0); + const uint64_t op = vgetq_lane_u64(wsOp, 1); +#else + const uint64_t whitespace = simd::ws_collector::impl(in01, whitespaceTableLocal); + const uint64_t op = simd::op_collector::impl(in01, opTable, spaceMask); +#endif + const uint64_t quotes = rope.quotes; + const uint64_t scalar = ~(op | whitespace | quotes); + const uint64_t follows = rope.followsNonquoteScalar(scalar); + const uint64_t scalarStart = scalar & ~follows; + return op | quotes | scalarStart; + } + + JSONIFIER_INLINE static uint64_t getStructurals(rope_type& __restrict rope, const simd_array_t in01, const jsonifier_simd_int_t opTable, + const jsonifier_simd_int_t spaceMask) noexcept { + const uint64_t op = simd::op_collector::impl(in01, opTable, spaceMask); + const uint64_t quotes = rope.quotes; + const uint64_t scalar = ~(op | quotes); + const uint64_t follows = rope.followsNonquoteScalar(scalar); + const uint64_t scalarStart = scalar & ~follows; + return op | quotes | scalarStart; + } + + JSONIFIER_INLINE unrestrict_t begin() noexcept { + tape[tapeCount] = static_cast(string_block_reader_locals::length); + return tape; + } + + JSONIFIER_INLINE unrestrict_t end() noexcept { + return tape + tapeCount; + } + + JSONIFIER_INLINE uint64_t getTapeCount() noexcept { + return tapeCount; + } + + JSONIFIER_INLINE ~simd_string_reader_locals() noexcept { + if (tape) { + allocator::deallocate(tape, capacity); + tape = nullptr; + } + } + + write_structural_index_ptr tape{}; + uint64_t tapeCount{}; + uint64_t capacity{}; + }; + +#if JSONIFIER_COMPILER_CLANG + using simd_string_reader = simd_string_reader_members; +#else + using simd_string_reader = simd_string_reader_locals; +#endif + + struct pod_simd_string_reader_members : simd::rope_detector, alloc_wrapper { + using allocator = alloc_wrapper; + using rope_type = simd::rope_detector; + static constexpr uint64_t stepBytes{ podBytesPerStep }; + + JSONIFIER_INLINE pod_simd_string_reader_members() noexcept = default; + + JSONIFIER_INLINE pod_simd_string_reader_members& operator=(pod_simd_string_reader_members&& other) noexcept { + if (&other != this) { + swap(tapeCount, other.tapeCount); + swap(capacity, other.capacity); + swap(length, other.length); + swap(tape, other.tape); + } + return *this; + } + + JSONIFIER_INLINE pod_simd_string_reader_members(pod_simd_string_reader_members&& other) noexcept : allocator{} { + *this = internal::move(other); + } + + pod_simd_string_reader_members& operator=(const pod_simd_string_reader_members& other) = delete; + pod_simd_string_reader_members(const pod_simd_string_reader_members& other) = delete; + + template + JSONIFIER_INLINE void processBlocksImpl(array& __restrict bitsArr, array& __restrict cntsArr, read_buffer_ptr blockPtr, + const simd_register_t bsRegister, const simd_register_t quoteRegister, const simd_register_t opTable, + const simd_register_t spaceMask, const simd_types... args) noexcept { + using simd_type = simd_register_t; + pod_simd_array_t inVals; + inVals.template set<0>(simd::gatherValuesU(blockPtr + currentBlock * simdBytesPerBlock)); + if constexpr (registerCount > 1) { + inVals.template set<1>(simd::gatherValuesU(blockPtr + currentBlock * simdBytesPerBlock + registerBytes * 1)); + if constexpr (registerCount > 2) { + inVals.template set<2>(simd::gatherValuesU(blockPtr + currentBlock * simdBytesPerBlock + registerBytes * 2)); + inVals.template set<3>(simd::gatherValuesU(blockPtr + currentBlock * simdBytesPerBlock + registerBytes * 3)); + } + } + rope_type::template nextScalar(inVals, bsRegister, quoteRegister); + const uint64_t structurals = getStructurals(inVals, opTable, spaceMask, args...) & ~rope_type::stringTail(); + bitsArr[currentBlock] = structurals; + cntsArr[currentBlock] = simd::tape_writer_op::correctedPopcount(structurals); + } + + template JSONIFIER_INLINE void resetDispatch(read_buffer_ptr rootIter, uint64_t stringLength) noexcept { + using simd_type = simd_register_t; + const uint64_t neededCapacity = stringLength + 64; + if (neededCapacity > capacity) { + auto newTape = allocator::allocate(neededCapacity); + if (tape) { + allocator::deallocate(tape, capacity); + } + tape = newTape; + capacity = neededCapacity; + } + + const simd_type bsRegister = simd::gatherValue('\\'); + const simd_type quoteRegister = simd::gatherValue('"'); + const simd_type opTable = simd::gatherValues(simd::opArray.data()); + const simd_type spaceMask = simd::gatherValue(static_cast(0x20)); + + if constexpr (minified) { + resetImpl(rootIter, stringLength, bsRegister, quoteRegister, opTable, spaceMask); + } else { + const simd_type whitespaceTableLocal = simd::gatherValues(simd::whitespaceArray.data()); + resetImpl(rootIter, stringLength, bsRegister, quoteRegister, opTable, spaceMask, whitespaceTableLocal); + } + } + + template JSONIFIER_INLINE void processBlocks(read_buffer_ptr blockPtr, + uint64_t stepBaseIndex, const simd_register_t bsRegister, const simd_register_t quoteRegister, + const simd_register_t opTable, const simd_register_t spaceMask, const simd_types... args) noexcept { + array bitsArr; + array cntsArr; + processBlocksImpl<0, registerBytes, registerCount>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + if constexpr (blocksPerStep > 1) { + processBlocksImpl<1, registerBytes, registerCount>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + if constexpr (blocksPerStep > 2) { + processBlocksImpl<2, registerBytes, registerCount>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + processBlocksImpl<3, registerBytes, registerCount>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + if constexpr (blocksPerStep > 4) { + processBlocksImpl<4, registerBytes, registerCount>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + processBlocksImpl<5, registerBytes, registerCount>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + processBlocksImpl<6, registerBytes, registerCount>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + processBlocksImpl<7, registerBytes, registerCount>(bitsArr, cntsArr, blockPtr, bsRegister, quoteRegister, opTable, spaceMask, args...); + } + } + } + + add_tape_values>::impl(bitsArr, cntsArr, tape + tapeCount, stepBaseIndex); + + tapeCount += cntsArr[0]; + if constexpr (blocksPerStep > 1) { + tapeCount += cntsArr[1]; + if constexpr (blocksPerStep > 2) { + tapeCount += cntsArr[2]; + tapeCount += cntsArr[3]; + if constexpr (blocksPerStep > 4) { + tapeCount += cntsArr[4]; + tapeCount += cntsArr[5]; + tapeCount += cntsArr[6]; + tapeCount += cntsArr[7]; + } + } + } + } + + template JSONIFIER_INLINE uint64_t getStructurals(const pod_simd_array_t in01, + const simd_register_t opTable, const simd_register_t spaceMask, const simd_register_t whitespaceTableLocal) noexcept { + const uint64_t whitespace = simd::pod_ws_collector::impl(in01, whitespaceTableLocal); + const uint64_t op = simd::scalar_op_collector::impl(in01, opTable, spaceMask); + const uint64_t scalar = ~(op | whitespace | rope_type::quotes); + const uint64_t follows = rope_type::followsNonquoteScalar(scalar); + const uint64_t scalarStart = scalar & ~follows; + return op | rope_type::quotes | scalarStart; + } + + template JSONIFIER_INLINE void resetImpl(read_buffer_ptr rootIter, uint64_t stringLength, + const simd_register_t bsRegister, const simd_register_t quoteRegister, const simd_register_t opTable, + const simd_register_t spaceMask, const simd_types... args) noexcept { + static constexpr uint64_t blocksPerStep{ stepBytes / simdBytesPerBlock }; + stringBlockReader.reset(rootIter, stringLength); + rope_type::prevInString = 0; + rope_type::prevScalar = 0; + rope_type::nextIsEscaped = 0; + tapeCount = 0; + if constexpr (registerCount * registerBytes == simdBytesPerBlock) { + while (stringBlockReader.hasFullStep()) { + const uint64_t stepBaseIndex = stringBlockReader.index(); + processBlocks(stringBlockReader.fullStep(), stepBaseIndex, bsRegister, quoteRegister, opTable, spaceMask, args...); + } + + while (stringBlockReader.hasFullBlock()) { + const uint64_t blockBaseIndex = stringBlockReader.index(); + processBlocks<1, registerBytes, registerCount>(stringBlockReader.fullBlock(), blockBaseIndex, bsRegister, quoteRegister, opTable, spaceMask, args...); + } + } + + if (stringBlockReader.getRemainderBytes() != 0) { + const uint64_t blockBaseIndex = stringBlockReader.index(); + processBlocks<1, registerBytes, registerCount>(stringBlockReader.getRemainder(), blockBaseIndex, bsRegister, quoteRegister, opTable, spaceMask, args...); + while (tapeCount > 0 && tape[tapeCount - 1] >= stringBlockReader.length()) { + --tapeCount; + } + } + length = stringBlockReader.length(); + } + + template JSONIFIER_INLINE uint64_t getStructurals(const pod_simd_array_t in01, + const simd_register_t opTable, const simd_register_t spaceMask) noexcept { + const uint64_t op = simd::scalar_op_collector::impl(in01, opTable, spaceMask); + const uint64_t scalar = ~(op | rope_type::quotes); + const uint64_t follows = rope_type::followsNonquoteScalar(scalar); + const uint64_t scalarStart = scalar & ~follows; + return op | rope_type::quotes | scalarStart; + } + + template JSONIFIER_INLINE void reset(read_buffer_ptr rootIter, uint64_t stringLength) noexcept { +#if JSONIFIER_ANY_AVX + if (stringLength <= 16) { + return resetDispatch(rootIter, stringLength); + } + if constexpr (simdBytesPerRegister >= 32) { + if (stringLength <= 32) { + return resetDispatch(rootIter, stringLength); + } + } +#elif JSONIFIER_NEON + if (stringLength <= 16) { + return resetDispatch(rootIter, stringLength); + } + if (stringLength <= 32) { + return resetDispatch(rootIter, stringLength); + } +#endif + resetDispatch(rootIter, stringLength); + } + + JSONIFIER_INLINE unrestrict_t begin() noexcept { + tape[tapeCount] = static_cast(length); + return tape; + } + + JSONIFIER_INLINE unrestrict_t end() noexcept { + return tape + tapeCount; + } + + JSONIFIER_INLINE uint64_t getTapeCount() noexcept { + return tapeCount; + } + + JSONIFIER_INLINE ~pod_simd_string_reader_members() noexcept { + if (tape) { + allocator::deallocate(tape, capacity); + tape = nullptr; + } + } + + pod_block_reader stringBlockReader; + write_structural_index_ptr tape{}; + uint64_t tapeCount{}; + uint64_t capacity{}; + uint64_t length{}; + }; + + using pod_simd_string_reader = pod_simd_string_reader_members; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/str_to_d.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/str_to_d.hpp new file mode 100644 index 000000000000..5df3d7a8ec0e --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/str_to_d.hpp @@ -0,0 +1,281 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/str_to_d.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::internal { + + consteval array generateExpTable() noexcept { + array table{}; + table[uint_c_v<'E'>] = true; + table[uint_c_v<'e'>] = true; + return table; + } + + template alignas(64) inline constexpr array expTableRaw = generateExpTable(); + + inline constexpr const bool* __restrict expTable{ expTableRaw<>.data() }; + + consteval array generateExpFracTable() noexcept { + array table{}; + table[uint_c_v<'.'>] = true; + table[uint_c_v<'E'>] = true; + table[uint_c_v<'e'>] = true; + return table; + } + + template alignas(64) inline constexpr array expFracTableRaw = generateExpFracTable(); + + inline constexpr const bool* __restrict expFracTable{ expFracTableRaw<>.data() }; + + enum class number_shapes : uint8_t { + general, + short_fraction, + long_integer, + }; + + struct number_shape_detector { + static constexpr uint64_t longIntegerThreshold{ 6 }; + static constexpr uint64_t shortFractionLimit{ 8 }; + + JSONIFIER_INLINE static number_shapes impl(read_buffer_ptr iter, read_buffer_ptr end) noexcept { + while (iter < end && ( *iter == uint_c_v<' '> || *iter == uint_c_v<'\t'> || *iter == uint_c_v<'\n'> || *iter == uint_c_v<'\r'> )) { + ++iter; + } + if (iter < end && *iter == uint_c_v<'-'>) { + ++iter; + } + read_buffer_ptr const integerStart = iter; + while (iter < end && isDigit(*iter)) { + ++iter; + } + if (static_cast(iter - integerStart) >= longIntegerThreshold) { + return number_shapes::long_integer; + } + if (iter >= end || *iter != uint_c_v<'.'>) { + return number_shapes::general; + } + ++iter; + read_buffer_ptr const fractionStart = iter; + while (iter < end && isDigit(*iter)) { + ++iter; + } + uint64_t const fractionDigits = static_cast(iter - fractionStart); + return (fractionDigits > 0 && fractionDigits < shortFractionLimit) ? number_shapes::short_fraction : number_shapes::general; + } + }; + + struct parsed_number { + read_buffer_ptr lastMatch{}; + span fraction{}; + span integer{}; + uint64_t mantissa{}; + int64_t exponent{}; + bool tooManyDigits{}; + bool negative{}; + bool valid{}; + }; + + template JSONIFIER_INLINE parsed_number parseNumberString(read_buffer_ptr iter, read_buffer_ptr end) noexcept { + parsed_number answer{}; + answer.negative = (*iter == uint_c_v<'-'>); + if (answer.negative) { + ++iter; + if (iter == end || !isDigit(*iter)) [[unlikely]] { + return answer; + } + } + read_buffer_ptr const startDigits = iter; + + uint64_t mantissa{}; + if ((iter != end) && isDigit(*iter)) { + mantissa = static_cast(*iter - uint_c_v<'0'>); + ++iter; + if constexpr (shape == number_shapes::long_integer) { + loopParseIfEightDigits(iter, end, mantissa); + while ((iter != end) && isDigit(*iter)) { + mantissa = 10 * mantissa + static_cast(*iter - uint_c_v<'0'>); + ++iter; + } + } else { + if ((iter != end) && isDigit(*iter)) { + mantissa = 10 * mantissa + static_cast(*iter - uint_c_v<'0'>); + ++iter; + if ((iter != end) && isDigit(*iter)) { + mantissa = 10 * mantissa + static_cast(*iter - uint_c_v<'0'>); + ++iter; + if ((iter != end) && isDigit(*iter)) { + mantissa = 10 * mantissa + static_cast(*iter - uint_c_v<'0'>); + ++iter; + if ((iter != end) && isDigit(*iter)) { + mantissa = 10 * mantissa + static_cast(*iter - uint_c_v<'0'>); + ++iter; + loopParseIfEightDigits(iter, end, mantissa); + while ((iter != end) && isDigit(*iter)) { + mantissa = 10 * mantissa + static_cast(*iter - uint_c_v<'0'>); + ++iter; + } + } + } + } + } + } + } + read_buffer_ptr const endOfIntegerPart = iter; + int64_t digitCount = static_cast(endOfIntegerPart - startDigits); + answer.integer = span{ startDigits, endOfIntegerPart }; + if (digitCount == 0 || (startDigits[0] == uint_c_v<'0'> && digitCount > 1)) [[unlikely]] { + return answer; + } + + int64_t exponent{}; + bool const hasDecimalPoint = (iter != end) && (*iter == uint_c_v<'.'>); + if (hasDecimalPoint) { + ++iter; + read_buffer_ptr const before = iter; + if constexpr (shape != number_shapes::short_fraction) { + loopParseIfEightDigits(iter, end, mantissa); + } + while ((iter != end) && isDigit(*iter)) { + uint8_t digit = static_cast(*iter - uint_c_v<'0'>); + ++iter; + mantissa = mantissa * 10 + digit; + } + exponent = before - iter; + answer.fraction = span{ before, iter }; + digitCount -= exponent; + if (exponent == 0) [[unlikely]] { + return answer; + } + } + + int64_t expNumber{}; + if ((iter != end) && (( *iter == uint_c_v<'e'> ) || ( *iter == uint_c_v<'E'> ))) { + ++iter; + bool negExp = false; + if ((iter != end) && ( *iter == uint_c_v<'-'> )) { + negExp = true; + ++iter; + } else if ((iter != end) && ( *iter == uint_c_v<'+'> )) { + ++iter; + } + if ((iter == end) || !isDigit(*iter)) [[unlikely]] { + return answer; + } + while ((iter != end) && isDigit(*iter)) { + uint8_t digit = static_cast(*iter - uint_c_v<'0'>); + if (expNumber < 0x10000000) { + expNumber = 10 * expNumber + digit; + } + ++iter; + } + if (negExp) { + expNumber = -expNumber; + } + exponent += expNumber; + } + answer.lastMatch = iter; + answer.valid = true; + + if (digitCount > 19) [[unlikely]] { + read_buffer_ptr start = startDigits; + while ((start != end) && ( *start == uint_c_v<'0'> || *start == uint_c_v<'.'> )) { + if (*start == uint_c_v<'0'>) { + --digitCount; + } + ++start; + } + if (digitCount > 19) { + answer.tooManyDigits = true; + static constexpr uint64_t minNineteenDigitInteger{ 1000000000000000000 }; + mantissa = 0; + iter = answer.integer.ptr; + while ((mantissa < minNineteenDigitInteger) && (iter != answer.integer.end)) { + mantissa = mantissa * 10 + static_cast(*iter - uint_c_v<'0'>); + ++iter; + } + if (mantissa >= minNineteenDigitInteger) { + exponent = endOfIntegerPart - iter + expNumber; + } else { + iter = answer.fraction.ptr; + while ((mantissa < minNineteenDigitInteger) && (iter != answer.fraction.end)) { + mantissa = mantissa * 10 + static_cast(*iter - uint_c_v<'0'>); + ++iter; + } + exponent = answer.fraction.ptr - iter + expNumber; + } + } + } + answer.exponent = exponent; + answer.mantissa = mantissa; + return answer; + } + + template struct float_parser { + template + JSONIFIER_INLINE static unrestrict_t parseFloat(value_type& value, read_buffer_ptr iter, read_buffer_ptr end = nullptr) noexcept { + if (iter >= end) [[unlikely]] { + return nullptr; + } + parsed_number pns = parseNumberString(iter, end); + if (!pns.valid) [[unlikely]] { + return nullptr; + } + if (!pns.tooManyDigits && clingerFastPath(pns.mantissa, pns.exponent, pns.negative, value)) [[likely]] { + return pns.lastMatch; + } + adjusted_mantissa am = computeFloat>(pns.exponent, pns.mantissa); + if (pns.tooManyDigits && am.power2 >= 0) [[unlikely]] { + if (am != computeFloat>(pns.exponent, pns.mantissa + 1)) { + am = computeError>(pns.exponent, pns.mantissa); + } + } + if (am.power2 < 0) [[unlikely]] { + am = digitComp(pns.integer, pns.fraction, pns.mantissa, pns.exponent, am); + } + if (am.power2 == binary_format::infinitePower) [[unlikely]] { + return nullptr; + } + toFloat(pns.negative, am, value); + return pns.lastMatch; + } + + JSONIFIER_INLINE static bool clingerFastPath(uint64_t mantissa, int64_t exponent, bool negative, value_type& value) noexcept { + if (binary_format::minExponentFastPath <= exponent && exponent <= binary_format::maxExponentFastPath && + mantissa <= binary_format::maxMantissaFastPathValue) { + if (rounds_to_nearest::roundsToNearest) { + value = static_cast(mantissa); + if (exponent < 0) { + value = value / binary_format::exactPowerOfTen(-exponent); + } else { + value = value * binary_format::exactPowerOfTen(exponent); + } + if (negative) { + value = -value; + } + return true; + } else if (exponent >= 0 && mantissa <= binary_format::maxMantissaFastPath(exponent)) { +#if JSONIFIER_COMPILER_CLANG + if (mantissa == 0) { + value = negative ? static_cast(-0.) : static_cast(0.); + return true; + } +#endif + value = static_cast(mantissa) * binary_format::exactPowerOfTen(exponent); + if (negative) { + value = -value; + } + return true; + } + } + return false; + } + }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/str_to_i.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/str_to_i.hpp new file mode 100644 index 000000000000..a4e6001c8e3d --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/str_to_i.hpp @@ -0,0 +1,1739 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/str_to_i.hpp + */ +#pragma once + +#include +#include +#include + +namespace jsonifier::internal { + +#if JSONIFIER_COMPILER_MSVC + + template inline constexpr uint64_t compValAddition{ [] { + if constexpr (negative) { + return 1ULL; + } else { + return 0ULL; + } + }() }; + + template consteval array, 256ULL> genRawCompVals() noexcept { + constexpr auto maxValue{ static_cast>(std::numeric_limits>::max()) + compValAddition }; + array, 256ULL> returnValues{}; + returnValues[uint_c_v<'0'>] = (maxValue - 0) / 10; + returnValues[uint_c_v<'1'>] = (maxValue - 1) / 10; + returnValues[uint_c_v<'2'>] = (maxValue - 2) / 10; + returnValues[uint_c_v<'3'>] = (maxValue - 3) / 10; + returnValues[uint_c_v<'4'>] = (maxValue - 4) / 10; + returnValues[uint_c_v<'5'>] = (maxValue - 5) / 10; + returnValues[uint_c_v<'6'>] = (maxValue - 6) / 10; + returnValues[uint_c_v<'7'>] = (maxValue - 7) / 10; + returnValues[uint_c_v<'8'>] = (maxValue - 8) / 10; + returnValues[uint_c_v<'9'>] = (maxValue - 9) / 10; + return returnValues; + } + + template alignas(64) inline constexpr auto compValsRaw{ genRawCompVals() }; + + template inline constexpr const std::make_unsigned_t* __restrict compVals{ compValsRaw.data() }; + + JSONIFIER_INLINE uint8_t peekByte(const uint8_t* iter, const uint8_t* end) noexcept { + return iter < end ? *iter : uint8_t{ 0 }; + } + + template JSONIFIER_INLINE uint8_t loadByte(const uint8_t* iter, const uint8_t* end) noexcept { + if constexpr (checked) { + return peekByte(iter, end); + } else { + return *iter; + } + } + + inline constexpr ptrdiff_t uncheckedIntegerSpan{ 21 }; + + inline constexpr int64_t maxFractionDigits{ 18 }; + + inline constexpr int64_t maxUnsignedFractionDigits{ 19 }; + + inline constexpr int64_t maxExponentValue{ 20 }; + + template consteval array generatePowerOfTen() noexcept { + array table{}; + value_type val = 1; + for (size_t i = 0; i < count; ++i) { + table[i] = val; + if (i + 1 < count) { + val *= 10; + } + } + return table; + } + + template struct powers_of_ten; + + template struct powers_of_ten { + alignas(64) static constexpr array valueRaw{ generatePowerOfTen() }; + + static constexpr const int64_t* __restrict value{ valueRaw.data() }; + }; + + template struct powers_of_ten { + alignas(64) static constexpr array valueRaw{ generatePowerOfTen() }; + + static constexpr const uint64_t* __restrict value{ valueRaw.data() }; + }; + + template struct integer_parser; + + template JSONIFIER_INLINE static value_type mul128Generic(value_type ab, value_type cd, value_type& hi) noexcept { + value_type aHigh = ab >> 32; + value_type aLow = ab & 0xFFFFFFFF; + value_type bHigh = cd >> 32; + value_type bLow = cd & 0xFFFFFFFF; + value_type loLo = aLow * bLow; + value_type loHi = aLow * bHigh; + value_type hiLo = aHigh * bLow; + value_type hiHi = aHigh * bHigh; + value_type cross = (loLo >> 32) + (loHi & 0xFFFFFFFF) + (hiLo & 0xFFFFFFFF); + value_type lo = (cross << 32) | (loLo & 0xFFFFFFFF); + hi = hiHi + (loHi >> 32) + (hiLo >> 32) + (cross >> 32); + return lo; + } + + template JSONIFIER_INLINE static bool multiply(value_type& value, value_type expValue) noexcept { + #if JSONIFIER_HAS_INT128 + const __int128_t res = static_cast<__int128_t>(value) * static_cast<__int128_t>(expValue); + value = static_cast(res); + return res <= std::numeric_limits::max(); + #elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_X64 + value_type values; + value = _mul128(value, expValue, &values); + return values == 0; + #elif JSONIFIER_COMPILER_CLANG || JSONIFIER_COMPILER_GCC + return !__builtin_mul_overflow(value, expValue, &value); + #else + value_type values; + value = mul128Generic(value, expValue, values); + return values == 0; + #endif + } + + template JSONIFIER_INLINE static bool multiply(value_type& value, value_type expValue) noexcept { + #if JSONIFIER_HAS_INT128 + const __uint128_t res = static_cast<__uint128_t>(value) * static_cast<__uint128_t>(expValue); + value = static_cast(res); + return res <= std::numeric_limits::max(); + #elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_X64 + value_type values; + value = _umul128(value, expValue, &values); + return values == 0; + #elif JSONIFIER_COMPILER_CLANG || JSONIFIER_COMPILER_GCC + return !__builtin_mul_overflow(value, expValue, &value); + #else + value_type values; + value = umul128Generic(value, expValue, values); + return values == 0; + #endif + } + + template JSONIFIER_INLINE static bool divide(value_type& value, value_type expValue) noexcept { + #if JSONIFIER_HAS_INT128 + const __int128_t dividend = static_cast<__int128_t>(value); + value = static_cast(dividend / static_cast<__int128_t>(expValue)); + return (dividend % static_cast<__int128_t>(expValue)) == 0; + #elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_X64 + value_type values; + value = _div128(0, value, expValue, &values); + return values == 0; + #else + value_type values; + values = value % expValue; + value = value / expValue; + return values == 0; + #endif + } + + template JSONIFIER_INLINE static bool divide(value_type& value, value_type expValue) noexcept { + #if JSONIFIER_HAS_INT128 + const __uint128_t dividend = static_cast<__uint128_t>(value); + value = static_cast(dividend / static_cast<__uint128_t>(expValue)); + return (dividend % static_cast<__uint128_t>(expValue)) == 0; + #elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_X64 + value_type values; + value = _udiv128(0, value, expValue, &values); + return values == 0; + #else + value_type values; + values = value % expValue; + value = value / expValue; + return values == 0; + #endif + } + + template struct integer_parser { + inline constexpr integer_parser() noexcept = default; + + template JSONIFIER_INLINE static const uint8_t* parseInteger(value_type& value, const uint8_t* iter, const uint8_t* end) noexcept { + using v_type_local = std::make_unsigned_t; + uint8_t numTmp{ loadByte(iter, end) }; + if (isDigit(numTmp)) [[likely]] { + value = static_cast(numTmp - uint_c_v<'0'>); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + return nullptr; + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (iter[-2] == uint_c_v<'0'>) [[unlikely]] { + return nullptr; + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + + if constexpr (uint_types) { + if (isDigit(numTmp)) [[likely]] { + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + } + + if (isDigit(numTmp)) [[likely]] { + if (static_cast(value) > static_cast(compVals[numTmp])) [[unlikely]] { + return nullptr; + } + value = static_cast(static_cast(value) * 10 + ( numTmp - uint_c_v<'0'> )); + ++iter; + numTmp = loadByte(iter, end); + } else [[unlikely]] { + if (!expFracTable[numTmp]) [[likely]] { + if constexpr (negative) { + value *= -1; + } + return iter; + } + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value *= -1, iter); + } else { + return finishParse(value, iter, end); + } + } + if (isDigit(numTmp)) [[unlikely]] { + return nullptr; + } + static constexpr v_type_local zeroVal{ 0 }; + if (expFracTable[numTmp]) [[unlikely]] { + if constexpr (negative) { + return (iter = finishParse(value, iter, end), value = static_cast(zeroVal - static_cast(value)), iter); + } else { + return finishParse(value, iter, end); + } + } + if constexpr (negative) { + value = static_cast(zeroVal - static_cast(value)); + } + return iter; + } + + JSONIFIER_INLINE static unrestrict_t parseInt(value_type& value, read_buffer_ptr iter, read_buffer_ptr end) noexcept { + if constexpr (int_types) { + if (iter < end) [[likely]] { + if (*iter == uint_c_v<'-'>) { + ++iter; + const uint8_t* resultPtr = (end - iter) >= uncheckedIntegerSpan ? parseInteger(value, iter, end) : parseInteger(value, iter, end); + if (resultPtr) [[likely]] { + return resultPtr; + } else { + value = 0; + return nullptr; + } + } else { + const uint8_t* resultPtr = + (end - iter) >= uncheckedIntegerSpan ? parseInteger(value, iter, end) : parseInteger(value, iter, end); + if (resultPtr) [[likely]] { + return resultPtr; + } else { + value = 0; + return nullptr; + } + } + } else { + value = 0; + return nullptr; + } + } else { + if (iter < end) [[likely]] { + const uint8_t* resultPtr = (end - iter) >= uncheckedIntegerSpan ? parseInteger(value, iter, end) : parseInteger(value, iter, end); + if (resultPtr) [[likely]] { + return resultPtr; + } else { + value = 0; + return nullptr; + } + } else { + value = 0; + return nullptr; + } + } + } + + JSONIFIER_INLINE static const uint8_t* parseExponent(value_type& value, const uint8_t* iter, int8_t expSign, const uint8_t* end) noexcept { + if (isDigit(peekByte(iter, end))) [[likely]] { + value_type expValue{ static_cast(peekByte(iter, end) - uint_c_v<'0'>) }; + ++iter; + while (isDigit(peekByte(iter, end))) { + expValue = expValue < maxExponentValue ? expValue * 10 + static_cast(peekByte(iter, end) - uint_c_v<'0'>) : expValue; + ++iter; + } + if (expValue < static_cast(powers_of_ten::valueRaw.size())) [[likely]] { + const value_type powerExp = powers_of_ten::value[expValue]; + static constexpr value_type doubleMax = std::numeric_limits::max(); + static constexpr value_type doubleMin = std::numeric_limits::min(); + expValue *= static_cast(expSign); + return (expSign > 0) ? ((value <= (doubleMax / powerExp)) ? (multiply(value, powerExp), iter) : nullptr) + : ((value / powerExp >= (doubleMin)) ? (divide(value, powerExp), iter) : nullptr); + } else [[unlikely]] { + return nullptr; + } + } else [[unlikely]] { + return nullptr; + } + } + + JSONIFIER_INLINE static const uint8_t* parseExponentPostFrac(value_type& value, const uint8_t* iter, int8_t expSign, value_type fracValue, + typename get_int_type::type fracDigits, const uint8_t* end) noexcept { + using exp_type = conditional_t, int64_t, value_type>; + if (isDigit(peekByte(iter, end))) [[likely]] { + exp_type expValue{ static_cast(peekByte(iter, end) - uint_c_v<'0'>) }; + ++iter; + while (isDigit(peekByte(iter, end))) { + expValue = expValue < maxExponentValue ? expValue * 10 + static_cast(peekByte(iter, end) - uint_c_v<'0'>) : expValue; + ++iter; + } + if (expValue < static_cast(powers_of_ten::valueRaw.size())) [[likely]] { + const value_type powerExp = powers_of_ten::value[expValue]; + + static constexpr value_type doubleMax = std::numeric_limits::max(); + static constexpr value_type doubleMin = std::numeric_limits::min(); + + if (fracDigits + expValue >= 0) { + expValue *= expSign; + const auto fractionalCorrection = expValue > fracDigits + ? fracValue * powers_of_ten::value[expValue - fracDigits] + : (fracDigits - expValue < static_cast(powers_of_ten::valueRaw.size()) + ? fracValue / powers_of_ten::value[fracDigits - expValue] + : value_type{}); + return (expSign > 0) ? ((value <= (doubleMax / powerExp)) + ? (multiply(value, powerExp), (fractionalCorrection <= doubleMax - value) ? (value += fractionalCorrection, iter) : nullptr) + : nullptr) + : ((value / powerExp >= (doubleMin)) ? (divide(value, powerExp), value += fractionalCorrection, iter) : nullptr); + } else { + return (expSign > 0) ? ((value <= (doubleMax / powerExp)) ? (multiply(value, powerExp), iter) : nullptr) + : ((value / powerExp >= (doubleMin)) ? (divide(value, powerExp), iter) : nullptr); + } + } else [[unlikely]] { + return nullptr; + } + } else [[unlikely]] { + return nullptr; + } + } + + JSONIFIER_INLINE static const uint8_t* parseFraction(value_type& value, const uint8_t* iter, const uint8_t* end) noexcept { + if (isDigit(peekByte(iter, end))) [[likely]] { + value_type fracValue{ static_cast(peekByte(iter, end) - uint_c_v<'0'>) }; + typename get_int_type::type fracDigits{ 1 }; + ++iter; + while (isDigit(peekByte(iter, end))) { + if (fracDigits < (uint_types ? maxUnsignedFractionDigits : maxFractionDigits)) { + fracValue = fracValue * 10 + static_cast(peekByte(iter, end) - uint_c_v<'0'>); + ++fracDigits; + } + ++iter; + } + if (expTable[peekByte(iter, end)]) { + ++iter; + int8_t expSign = 1; + if (peekByte(iter, end) == uint_c_v<'-'>) { + expSign = -1; + ++iter; + } else if (peekByte(iter, end) == uint_c_v<'+'>) { + ++iter; + } + return parseExponentPostFrac(value, iter, expSign, fracValue, fracDigits, end); + } + } + if (!expFracTable[peekByte(iter, end)]) [[likely]] { + return iter; + } else { + return nullptr; + } + } + + static const uint8_t* finishParse(value_type& value, const uint8_t* iter, const uint8_t* end) noexcept { + if (peekByte(iter, end) == uint_c_v<'.'>) [[unlikely]] { + ++iter; + return parseFraction(value, iter, end); + } else if (expTable[peekByte(iter, end)]) { + ++iter; + int8_t expSign = 1; + if (peekByte(iter, end) == uint_c_v<'-'>) { + expSign = -1; + ++iter; + } else if (peekByte(iter, end) == uint_c_v<'+'>) { + ++iter; + } + return parseExponent(value, iter, expSign, end); + } + if (!expFracTable[peekByte(iter, end)]) [[likely]] { + return nullptr; + } else { + return nullptr; + } + } + }; + +#else + +template inline constexpr uint64_t compValAddition{ [] { + if constexpr (negative) { + return 1ULL; + } else { + return 0ULL; + } +}() }; + +template consteval array, 256ULL> genRawCompVals() noexcept { + constexpr auto maxValue{ static_cast>(std::numeric_limits>::max()) + compValAddition }; + array, 256ULL> returnValues{}; + returnValues[uint_c_v<'0'>] = (maxValue - 0) / 10; + returnValues[uint_c_v<'1'>] = (maxValue - 1) / 10; + returnValues[uint_c_v<'2'>] = (maxValue - 2) / 10; + returnValues[uint_c_v<'3'>] = (maxValue - 3) / 10; + returnValues[uint_c_v<'4'>] = (maxValue - 4) / 10; + returnValues[uint_c_v<'5'>] = (maxValue - 5) / 10; + returnValues[uint_c_v<'6'>] = (maxValue - 6) / 10; + returnValues[uint_c_v<'7'>] = (maxValue - 7) / 10; + returnValues[uint_c_v<'8'>] = (maxValue - 8) / 10; + returnValues[uint_c_v<'9'>] = (maxValue - 9) / 10; + return returnValues; +} + +template alignas(64) inline constexpr auto compValsRaw{ genRawCompVals() }; + +template inline constexpr const std::make_unsigned_t* __restrict compVals{ compValsRaw.data() }; + +namespace int_swar { + + template inline constexpr v_type repeatBytes = + static_cast(static_cast(0x0101010101010101ull) * static_cast(repeat)); + + template JSONIFIER_INLINE v_type countZeros(v_type value) noexcept { + return static_cast(simd::countrZeroUnsafe(value)); + } + + template struct parse_chunk_result { + v_type value; + uint64_t digits; + }; + + template JSONIFIER_INLINE v_type load(read_buffer_ptr str) noexcept { + v_type chunk; + pow2MemcpyWrapper(&chunk, str); + if constexpr (std::endian::native == std::endian::big) { + chunk = internal::byteswap(chunk); + } + return chunk; + } + + template JSONIFIER_INLINE v_type load(read_buffer_ptr str) noexcept { + return *str; + } + + template JSONIFIER_INLINE v_type mask(v_type raw) noexcept { + static constexpr v_type high{ repeatBytes(0x80), v_type> }; + static constexpr v_type low{ repeatBytes(0x7F), v_type> }; + static constexpr v_type up{ repeatBytes(0x46), v_type> }; + static constexpr v_type down{ repeatBytes(0x50), v_type> }; + const v_type body{ static_cast(raw & low) }; + const v_type above{ static_cast((body + up) & high) }; + const v_type atLeast{ static_cast((body + down) & high) }; + return static_cast((raw & high) | above | (~atLeast & high)); + } + + JSONIFIER_INLINE bool incorrect(uint8_t raw) noexcept { + return static_cast(raw - static_cast(0x30)) > 9u; + } + + template struct fold; + + template struct fold { + JSONIFIER_INLINE static uint64_t impl(v_type raw) noexcept { + const v_type sub{ raw - repeatBytes(0x30), v_type> }; + v_type val = (sub * 10 + (sub >> 8)) & 0x00FF00FF00FF00FFULL; + val = (val * 100 + (val >> 16)) & 0x0000FFFF0000FFFFULL; + return (val * 10000 + (val >> 32)) & 0x00000000FFFFFFFFULL; + } + }; + + template struct fold { + JSONIFIER_INLINE static uint64_t impl(v_type raw) noexcept { + const v_type sub{ static_cast(raw - repeatBytes(0x30), v_type>) }; + v_type val = (sub * 10 + (sub >> 8)) & 0x00FF00FFUL; + return static_cast((val * 100 + (val >> 16)) & 0x0000FFFFUL); + } + }; + + template struct fold { + JSONIFIER_INLINE static uint64_t impl(v_type raw) noexcept { + const v_type sub{ static_cast(raw - repeatBytes(0x30), v_type>) }; + return static_cast((sub & 0xFFU) * 10 + (sub >> 8)); + } + }; + + template struct fold { + JSONIFIER_INLINE static uint64_t impl(v_type raw) noexcept { + return static_cast(static_cast(raw - static_cast(0x30))); + } + }; + + template struct parse_fixed; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint8_t s1 = load(str); + if (incorrect(s1)) [[unlikely]] { + return { 0, 0 }; + } + return { static_cast(fold::impl(s1)), 1 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint16_t s2 = load(str); + const uint16_t m2 = mask(s2); + if (m2) [[unlikely]] { + return { 0, static_cast(countZeros(m2) >> 3) }; + } + return { static_cast(fold::impl(s2)), 2 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint16_t s2 = load(str); + const uint8_t s1 = load(str + 2); + const uint16_t m2 = mask(s2); + if (m2) [[unlikely]] { + return { 0, static_cast(countZeros(m2) >> 3) }; + } + if (incorrect(s1)) [[unlikely]] { + return { 0, 2 }; + } + return { static_cast(fold::impl(s2) * 10ULL + fold::impl(s1)), 3 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint32_t s4 = load(str); + const uint32_t m4 = mask(s4); + if (m4) [[unlikely]] { + return { 0, static_cast(countZeros(m4) >> 3) }; + } + return { static_cast(fold::impl(s4)), 4 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint32_t s4 = load(str); + const uint8_t s1 = load(str + 4); + const uint32_t m4 = mask(s4); + if (m4) [[unlikely]] { + return { 0, static_cast(countZeros(m4) >> 3) }; + } + if (incorrect(s1)) [[unlikely]] { + return { 0, 4 }; + } + return { static_cast(fold::impl(s4) * 10ULL + fold::impl(s1)), 5 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint32_t s4 = load(str); + const uint16_t s2 = load(str + 4); + const uint32_t m4 = mask(s4); + if (m4) [[unlikely]] { + return { 0, static_cast(countZeros(m4) >> 3) }; + } + const uint16_t m2 = mask(s2); + if (m2) [[unlikely]] { + return { 0, 4ULL + static_cast(countZeros(m2) >> 3) }; + } + return { static_cast(fold::impl(s4) * 100ULL + fold::impl(s2)), 6 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint32_t s4 = load(str); + const uint16_t s2 = load(str + 4); + const uint8_t s1 = load(str + 6); + const uint32_t m4 = mask(s4); + if (m4) [[unlikely]] { + return { 0, static_cast(countZeros(m4) >> 3) }; + } + const uint16_t m2 = mask(s2); + if (m2) [[unlikely]] { + return { 0, 4ULL + static_cast(countZeros(m2) >> 3) }; + } + if (incorrect(s1)) [[unlikely]] { + return { 0, 6 }; + } + return { static_cast(fold::impl(s4) * 1000ULL + fold::impl(s2) * 10ULL + fold::impl(s1)), 7 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint64_t s8 = load(str); + const uint64_t m8 = mask(s8); + if (m8) [[unlikely]] { + return { 0, countZeros(m8) >> 3 }; + } + return { static_cast(fold::impl(s8)), 8 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint64_t s8 = load(str); + const uint8_t s1 = load(str + 8); + const uint64_t m8 = mask(s8); + if (m8) [[unlikely]] { + return { 0, countZeros(m8) >> 3 }; + } + if (incorrect(s1)) [[unlikely]] { + return { 0, 8 }; + } + return { static_cast(fold::impl(s8) * 10ULL + fold::impl(s1)), 9 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint64_t s8 = load(str); + const uint16_t s2 = load(str + 8); + const uint64_t m8 = mask(s8); + if (m8) [[unlikely]] { + return { 0, countZeros(m8) >> 3 }; + } + const uint16_t m2 = mask(s2); + if (m2) [[unlikely]] { + return { 0, 8ULL + static_cast(countZeros(m2) >> 3) }; + } + return { static_cast(fold::impl(s8) * 100ULL + fold::impl(s2)), 10 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint64_t s8 = load(str); + const uint16_t s2 = load(str + 8); + const uint8_t s1 = load(str + 10); + const uint64_t m8 = mask(s8); + if (m8) [[unlikely]] { + return { 0, countZeros(m8) >> 3 }; + } + const uint16_t m2 = mask(s2); + if (m2) [[unlikely]] { + return { 0, 8ULL + static_cast(countZeros(m2) >> 3) }; + } + if (incorrect(s1)) [[unlikely]] { + return { 0, 10 }; + } + return { static_cast(fold::impl(s8) * 1000ULL + fold::impl(s2) * 10ULL + fold::impl(s1)), 11 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint64_t s8 = load(str); + const uint32_t s4 = load(str + 8); + const uint64_t m8 = mask(s8); + if (m8) [[unlikely]] { + return { 0, countZeros(m8) >> 3 }; + } + const uint32_t m4 = mask(s4); + if (m4) [[unlikely]] { + return { 0, 8ULL + static_cast(countZeros(m4) >> 3) }; + } + return { static_cast(fold::impl(s8) * 10000ULL + fold::impl(s4)), 12 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint64_t s8 = load(str); + const uint32_t s4 = load(str + 8); + const uint8_t s1 = load(str + 12); + const uint64_t m8 = mask(s8); + if (m8) [[unlikely]] { + return { 0, countZeros(m8) >> 3 }; + } + const uint32_t m4 = mask(s4); + if (m4) [[unlikely]] { + return { 0, 8ULL + static_cast(countZeros(m4) >> 3) }; + } + if (incorrect(s1)) [[unlikely]] { + return { 0, 12 }; + } + return { static_cast(fold::impl(s8) * 100000ULL + fold::impl(s4) * 10ULL + fold::impl(s1)), 13 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint64_t s8 = load(str); + const uint32_t s4 = load(str + 8); + const uint16_t s2 = load(str + 12); + const uint64_t m8 = mask(s8); + if (m8) [[unlikely]] { + return { 0, countZeros(m8) >> 3 }; + } + const uint32_t m4 = mask(s4); + if (m4) [[unlikely]] { + return { 0, 8ULL + static_cast(countZeros(m4) >> 3) }; + } + const uint16_t m2 = mask(s2); + if (m2) [[unlikely]] { + return { 0, 12ULL + static_cast(countZeros(m2) >> 3) }; + } + return { static_cast(fold::impl(s8) * 1000000ULL + fold::impl(s4) * 100ULL + fold::impl(s2)), 14 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint64_t s8 = load(str); + const uint32_t s4 = load(str + 8); + const uint16_t s2 = load(str + 12); + const uint8_t s1 = load(str + 14); + const uint64_t m8 = mask(s8); + if (m8) [[unlikely]] { + return { 0, countZeros(m8) >> 3 }; + } + const uint32_t m4 = mask(s4); + if (m4) [[unlikely]] { + return { 0, 8ULL + static_cast(countZeros(m4) >> 3) }; + } + const uint16_t m2 = mask(s2); + if (m2) [[unlikely]] { + return { 0, 12ULL + static_cast(countZeros(m2) >> 3) }; + } + if (incorrect(s1)) [[unlikely]] { + return { 0, 14 }; + } + return { static_cast(fold::impl(s8) * 10000000ULL + fold::impl(s4) * 1000ULL + fold::impl(s2) * 10ULL + fold::impl(s1)), + 15 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint64_t s8a = load(str); + const uint64_t s8b = load(str + 8); + const uint64_t m8a = mask(s8a); + if (m8a) [[unlikely]] { + return { 0, countZeros(m8a) >> 3 }; + } + const uint64_t m8b = mask(s8b); + if (m8b) [[unlikely]] { + return { 0, 8ULL + (countZeros(m8b) >> 3) }; + } + return { static_cast(fold::impl(s8a) * 100000000ULL + fold::impl(s8b)), 16 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint64_t s8a = load(str); + const uint64_t s8b = load(str + 8); + const uint8_t s1 = load(str + 16); + const uint64_t m8a = mask(s8a); + if (m8a) [[unlikely]] { + return { 0, countZeros(m8a) >> 3 }; + } + const uint64_t m8b = mask(s8b); + if (m8b) [[unlikely]] { + return { 0, 8ULL + (countZeros(m8b) >> 3) }; + } + if (incorrect(s1)) [[unlikely]] { + return { 0, 16 }; + } + return { static_cast(fold::impl(s8a) * 1000000000ULL + fold::impl(s8b) * 10ULL + fold::impl(s1)), 17 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint64_t s8a = load(str); + const uint64_t s8b = load(str + 8); + const uint16_t s2 = load(str + 16); + const uint64_t m8a = mask(s8a); + if (m8a) [[unlikely]] { + return { 0, countZeros(m8a) >> 3 }; + } + const uint64_t m8b = mask(s8b); + if (m8b) [[unlikely]] { + return { 0, 8ULL + (countZeros(m8b) >> 3) }; + } + const uint16_t m2 = mask(s2); + if (m2) [[unlikely]] { + return { 0, 16ULL + static_cast(countZeros(m2) >> 3) }; + } + return { static_cast(fold::impl(s8a) * 10000000000ULL + fold::impl(s8b) * 100ULL + fold::impl(s2)), 18 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint64_t s8a = load(str); + const uint64_t s8b = load(str + 8); + const uint16_t s2 = load(str + 16); + const uint8_t s1 = load(str + 18); + const uint64_t m8a = mask(s8a); + if (m8a) [[unlikely]] { + return { 0, countZeros(m8a) >> 3 }; + } + const uint64_t m8b = mask(s8b); + if (m8b) [[unlikely]] { + return { 0, 8ULL + (countZeros(m8b) >> 3) }; + } + const uint16_t m2 = mask(s2); + if (m2) [[unlikely]] { + return { 0, 16ULL + static_cast(countZeros(m2) >> 3) }; + } + if (incorrect(s1)) [[unlikely]] { + return { 0, 18 }; + } + return { static_cast( + fold::impl(s8a) * 100000000000ULL + fold::impl(s8b) * 1000ULL + fold::impl(s2) * 10ULL + fold::impl(s1)), + 19 }; + } + }; + + template struct parse_fixed { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str) noexcept { + const uint64_t s8a = load(str); + const uint64_t s8b = load(str + 8); + const uint32_t s4 = load(str + 16); + const uint64_t m8a = mask(s8a); + if (m8a) [[unlikely]] { + return { 0, countZeros(m8a) >> 3 }; + } + const uint64_t m8b = mask(s8b); + if (m8b) [[unlikely]] { + return { 0, 8ULL + (countZeros(m8b) >> 3) }; + } + const uint32_t m4 = mask(s4); + if (m4) [[unlikely]] { + return { 0, 16ULL + static_cast(countZeros(m4) >> 3) }; + } + return { static_cast(fold::impl(s8a) * 1000000000000ULL + fold::impl(s8b) * 10000ULL + fold::impl(s4)), 20 }; + } + }; + + template struct dispatch_table; + + template struct dispatch_table { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str, uint64_t length) noexcept { + switch (length) { + case 1: + return parse_fixed::impl(str); + case 2: + return parse_fixed::impl(str); + case 3: + return parse_fixed::impl(str); + case 4: + return parse_fixed::impl(str); + case 5: + return parse_fixed::impl(str); + case 6: + return parse_fixed::impl(str); + case 7: + return parse_fixed::impl(str); + case 8: + return parse_fixed::impl(str); + case 9: + return parse_fixed::impl(str); + case 10: + return parse_fixed::impl(str); + case 11: + return parse_fixed::impl(str); + case 12: + return parse_fixed::impl(str); + case 13: + return parse_fixed::impl(str); + case 14: + return parse_fixed::impl(str); + case 15: + return parse_fixed::impl(str); + case 16: + return parse_fixed::impl(str); + case 17: + return parse_fixed::impl(str); + case 18: + return parse_fixed::impl(str); + default: + return parse_fixed::impl(str); + } + } + }; + + template struct dispatch_table { + JSONIFIER_INLINE static parse_chunk_result impl(read_buffer_ptr str, uint64_t length) noexcept { + switch (length) { + case 1: + return parse_fixed::impl(str); + case 2: + return parse_fixed::impl(str); + case 3: + return parse_fixed::impl(str); + case 4: + return parse_fixed::impl(str); + case 5: + return parse_fixed::impl(str); + case 6: + return parse_fixed::impl(str); + case 7: + return parse_fixed::impl(str); + case 8: + return parse_fixed::impl(str); + case 9: + return parse_fixed::impl(str); + case 10: + return parse_fixed::impl(str); + case 11: + return parse_fixed::impl(str); + case 12: + return parse_fixed::impl(str); + case 13: + return parse_fixed::impl(str); + case 14: + return parse_fixed::impl(str); + case 15: + return parse_fixed::impl(str); + case 16: + return parse_fixed::impl(str); + case 17: + return parse_fixed::impl(str); + case 18: + return parse_fixed::impl(str); + case 19: + return parse_fixed::impl(str); + default: + return parse_fixed::impl(str); + } + } + }; + + template JSONIFIER_INLINE bool exceedsLimit(v_type value, uint8_t last) noexcept { + return static_cast(value) > static_cast(compVals[last]); + } + +} + +template JSONIFIER_INLINE unrestrict_t parseSwarDigits(value_type& value, read_buffer_ptr iter, read_buffer_ptr end) noexcept { + static_assert(sizeof(value_type) == 8, "parseSwarDigits only supports 64-bit integers."); + using v_type_local = std::make_unsigned_t; + static constexpr uint64_t maxDigits{ uint_types ? 20ULL : 19ULL }; + + if (iter >= end || !isDigit(*iter)) [[unlikely]] { + return nullptr; + } + if (*iter == uint_c_v<'0'> && iter + 1 < end && isDigit(iter[1])) [[unlikely]] { + return nullptr; + } + + const uint64_t avail{ static_cast(end - iter) }; + const uint64_t length{ avail < maxDigits ? avail : maxDigits }; + int_swar::parse_chunk_result res{}; + + if (length == maxDigits) { + res = int_swar::dispatch_table::impl(iter, length - 1); + if (res.digits == length - 1) { + const uint8_t last{ iter[length - 1] }; + if (!int_swar::incorrect(last)) { + if (static_cast(res.value) > static_cast(compVals[last])) [[unlikely]] { + return nullptr; + } + res.value = static_cast(static_cast(res.value) * 10 + static_cast(last - uint_c_v<'0'>)); + res.digits = length; + } + } + } else { + res = int_swar::dispatch_table::impl(iter, length); + } + + if (res.digits != length) { + res = int_swar::dispatch_table::impl(iter, res.digits); + } + + iter += res.digits; + if (iter < end && isDigit(*iter)) [[unlikely]] { + return nullptr; + } + value = res.value; + return iter; +} + +template consteval array genPowerOfTenTable() noexcept { + array returnValues{}; + value_type value{ 1 }; + for (uint64_t x = 0; x < size; ++x) { + returnValues[x] = value; + if (x + 1 < size) { + value *= 10; + } + } + return returnValues; +} + +template struct powers_of_ten; + +template struct powers_of_ten { + alignas(64) static constexpr array valueRaw{ genPowerOfTenTable() }; + + static constexpr const int64_t* __restrict value{ valueRaw.data() }; +}; + +template struct powers_of_ten { + alignas(64) static constexpr array valueRaw{ genPowerOfTenTable() }; + + static constexpr const uint64_t* __restrict value{ valueRaw.data() }; +}; + +JSONIFIER_INLINE uint8_t peekByte(read_buffer_ptr iter, read_buffer_ptr end) noexcept { + return iter < end ? *iter : uint8_t{}; +} + +JSONIFIER_INLINE unrestrict_t parseExponentDigits(int64_t& expValue, read_buffer_ptr iter, read_buffer_ptr end) noexcept { + expValue = 0; + while (isDigit(peekByte(iter, end))) { + if (expValue < 1000) { + expValue = expValue * 10 + (*iter - uint_c_v<'0'>); + } + ++iter; + } + return iter; +} + +template struct integer_parser; + +template struct integer_parser { + template JSONIFIER_INLINE static unrestrict_t parseInteger(value_type& value, read_buffer_ptr iter, read_buffer_ptr end) noexcept { + using v_type_local = std::make_unsigned_t; + iter = parseSwarDigits(value, iter, end); + if (!iter) [[unlikely]] { + return nullptr; + } + if (iter < end && expFracTable[*iter]) [[unlikely]] { + iter = finishParse(value, iter, end); + } + if constexpr (negative) { + value = static_cast(static_cast(0) - static_cast(value)); + } + return iter; + } + + JSONIFIER_INLINE static unrestrict_t parseExponent(value_type& value, read_buffer_ptr iter, read_buffer_ptr end, int8_t expSign) noexcept { + if (!isDigit(peekByte(iter, end))) [[unlikely]] { + return nullptr; + } + int64_t expValue; + iter = parseExponentDigits(expValue, iter, end); + if (expValue >= 19) [[unlikely]] { + return nullptr; + } + const value_type powerExp = powers_of_ten::value[expValue]; + static constexpr value_type doubleMax = std::numeric_limits::max(); + static constexpr value_type doubleMin = std::numeric_limits::min(); + return (expSign > 0) ? ((value <= (doubleMax / powerExp)) ? (multiply(value, powerExp), iter) : nullptr) + : ((value / powerExp >= (doubleMin)) ? (divide(value, powerExp), iter) : nullptr); + } + + JSONIFIER_INLINE static unrestrict_t parseExponentPostFrac(value_type& value, read_buffer_ptr iter, read_buffer_ptr end, int8_t expSign, value_type fracValue, + int64_t fracDigits) noexcept { + if (!isDigit(peekByte(iter, end))) [[unlikely]] { + return nullptr; + } + int64_t expValue; + iter = parseExponentDigits(expValue, iter, end); + if (expValue >= 19) [[unlikely]] { + return nullptr; + } + const value_type powerExp = powers_of_ten::value[expValue]; + static constexpr value_type doubleMax = std::numeric_limits::max(); + static constexpr value_type doubleMin = std::numeric_limits::min(); + const int64_t shift = expSign * expValue - fracDigits; + const value_type fractionalCorrection = + shift >= 0 ? fracValue * powers_of_ten::value[shift] : (-shift < 19 ? fracValue / powers_of_ten::value[-shift] : value_type{}); + if (expSign > 0) { + if (value > doubleMax / powerExp) [[unlikely]] { + return nullptr; + } + multiply(value, powerExp); + if (fractionalCorrection > doubleMax - value) [[unlikely]] { + return nullptr; + } + value += fractionalCorrection; + return iter; + } + if (value / powerExp < doubleMin) [[unlikely]] { + return nullptr; + } + divide(value, powerExp); + value += fractionalCorrection; + return iter; + } + + JSONIFIER_INLINE static unrestrict_t parseFraction(value_type& value, read_buffer_ptr iter, read_buffer_ptr end) noexcept { + if (isDigit(peekByte(iter, end))) [[likely]] { + value_type fracValue{}; + int64_t fracDigits{}; + while (isDigit(peekByte(iter, end))) { + if (fracDigits < 18) { + fracValue = fracValue * 10 + static_cast(*iter - uint_c_v<'0'>); + ++fracDigits; + } + ++iter; + } + if (expTable[peekByte(iter, end)]) { + ++iter; + int8_t expSign = 1; + if (peekByte(iter, end) == uint_c_v<'-'>) { + expSign = -1; + ++iter; + } else if (peekByte(iter, end) == uint_c_v<'+'>) { + ++iter; + } + return parseExponentPostFrac(value, iter, end, expSign, fracValue, fracDigits); + } + } + if (!expFracTable[peekByte(iter, end)]) [[likely]] { + return iter; + } else { + return nullptr; + } + } + + JSONIFIER_INLINE static unrestrict_t finishParse(value_type& value, read_buffer_ptr iter, read_buffer_ptr end) noexcept { + if (peekByte(iter, end) == uint_c_v<'.'>) [[unlikely]] { + ++iter; + return parseFraction(value, iter, end); + } else if (expTable[peekByte(iter, end)]) { + ++iter; + int8_t expSign = 1; + if (peekByte(iter, end) == uint_c_v<'-'>) { + expSign = -1; + ++iter; + } else if (peekByte(iter, end) == uint_c_v<'+'>) { + ++iter; + } + return parseExponent(value, iter, end, expSign); + } + if (!expFracTable[peekByte(iter, end)]) [[likely]] { + return nullptr; + } else { + return nullptr; + } + } + + JSONIFIER_INLINE static unrestrict_t parseInt(value_type& value, read_buffer_ptr iter, read_buffer_ptr end) noexcept { + if (iter < end) [[likely]] { + if (*iter == uint_c_v<'-'>) { + ++iter; + read_buffer_ptr resultPtr = parseInteger(value, std::bit_cast(iter), std::bit_cast(end)); + if (resultPtr) [[likely]] { + iter += resultPtr - std::bit_cast(iter); + return iter; + } else { + value = 0; + return nullptr; + } + } else { + read_buffer_ptr resultPtr = parseInteger(value, std::bit_cast(iter), std::bit_cast(end)); + if (resultPtr) [[likely]] { + iter += resultPtr - std::bit_cast(iter); + return iter; + } else { + value = 0; + return nullptr; + } + } + } else { + value = 0; + return nullptr; + } + } + + JSONIFIER_INLINE static value_type mul128Generic(value_type ab, value_type cd, value_type& hi) noexcept { + value_type aHigh = ab >> 32; + value_type aLow = ab & 0xFFFFFFFF; + value_type bHigh = cd >> 32; + value_type bLow = cd & 0xFFFFFFFF; + value_type loLo = aLow * bLow; + value_type loHi = aLow * bHigh; + value_type hiLo = aHigh * bLow; + value_type hiHi = aHigh * bHigh; + value_type cross = (loLo >> 32) + (loHi & 0xFFFFFFFF) + (hiLo & 0xFFFFFFFF); + value_type lo = (cross << 32) | (loLo & 0xFFFFFFFF); + hi = hiHi + (loHi >> 32) + (hiLo >> 32) + (cross >> 32); + return lo; + } + + JSONIFIER_INLINE static bool multiply(value_type& value, value_type expValue) noexcept { + #if JSONIFIER_HAS_INT128 + const __int128_t res = static_cast<__int128_t>(value) * static_cast<__int128_t>(expValue); + value = static_cast(res); + return res <= std::numeric_limits::max(); + #elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_X64 + value_type values; + value = _mul128(value, expValue, &values); + return values == 0; + #elif JSONIFIER_COMPILER_CLANG || JSONIFIER_COMPILER_GCC + return !__builtin_mul_overflow(value, expValue, &value); + #else + value_type values; + value = mul128Generic(value, expValue, values); + return values == 0; + #endif + } + + JSONIFIER_INLINE static bool divide(value_type& value, value_type expValue) noexcept { + #if JSONIFIER_HAS_INT128 + const __int128_t dividend = static_cast<__int128_t>(value); + value = static_cast(dividend / static_cast<__int128_t>(expValue)); + return (dividend % static_cast<__int128_t>(expValue)) == 0; + #elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_X64 + value_type values; + value = _div128(0, value, expValue, &values); + return values == 0; + #else + value_type values; + values = value % expValue; + value = value / expValue; + return values == 0; + #endif + } +}; + +template struct integer_parser { + JSONIFIER_INLINE static unrestrict_t parseExponent(value_type& value, read_buffer_ptr iter, read_buffer_ptr end, int8_t expSign) noexcept { + if (!isDigit(peekByte(iter, end))) [[unlikely]] { + return nullptr; + } + int64_t expValue; + iter = parseExponentDigits(expValue, iter, end); + if (expValue > 19) [[unlikely]] { + return nullptr; + } + const value_type powerExp = powers_of_ten::value[expValue]; + static constexpr value_type doubleMax = std::numeric_limits::max(); + static constexpr value_type doubleMin = std::numeric_limits::min(); + return (expSign > 0) ? ((value <= (doubleMax / powerExp)) ? (multiply(value, powerExp), iter) : nullptr) + : ((value / powerExp >= (doubleMin)) ? (divide(value, powerExp), iter) : nullptr); + } + + JSONIFIER_INLINE static unrestrict_t parseExponentPostFrac(value_type& value, read_buffer_ptr iter, read_buffer_ptr end, int8_t expSign, value_type fracValue, + int64_t fracDigits) noexcept { + if (!isDigit(peekByte(iter, end))) [[unlikely]] { + return nullptr; + } + int64_t expValue; + iter = parseExponentDigits(expValue, iter, end); + if (expValue > 19) [[unlikely]] { + return nullptr; + } + const value_type powerExp = powers_of_ten::value[expValue]; + static constexpr value_type doubleMax = std::numeric_limits::max(); + static constexpr value_type doubleMin = std::numeric_limits::min(); + const int64_t shift = expSign * expValue - fracDigits; + const value_type fractionalCorrection = + shift >= 0 ? fracValue * powers_of_ten::value[shift] : (-shift < 20 ? fracValue / powers_of_ten::value[-shift] : value_type{}); + if (expSign > 0) { + if (value > doubleMax / powerExp) [[unlikely]] { + return nullptr; + } + multiply(value, powerExp); + if (fractionalCorrection > doubleMax - value) [[unlikely]] { + return nullptr; + } + value += fractionalCorrection; + return iter; + } + if (value / powerExp < doubleMin) [[unlikely]] { + return nullptr; + } + divide(value, powerExp); + value += fractionalCorrection; + return iter; + } + + JSONIFIER_INLINE static unrestrict_t parseFraction(value_type& value, read_buffer_ptr iter, read_buffer_ptr end) noexcept { + if (isDigit(peekByte(iter, end))) [[likely]] { + value_type fracValue{}; + int64_t fracDigits{}; + while (isDigit(peekByte(iter, end))) { + if (fracDigits < 19) { + fracValue = fracValue * 10 + static_cast(*iter - uint_c_v<'0'>); + ++fracDigits; + } + ++iter; + } + if (expTable[peekByte(iter, end)]) { + ++iter; + int8_t expSign = 1; + if (peekByte(iter, end) == uint_c_v<'-'>) { + expSign = -1; + ++iter; + } else if (peekByte(iter, end) == uint_c_v<'+'>) { + ++iter; + } + return parseExponentPostFrac(value, iter, end, expSign, fracValue, fracDigits); + } + } + if (!expFracTable[peekByte(iter, end)]) [[likely]] { + return iter; + } else { + return nullptr; + } + } + + JSONIFIER_INLINE static unrestrict_t parseInteger(value_type& value, read_buffer_ptr iter, read_buffer_ptr end) noexcept { + iter = parseSwarDigits(value, iter, end); + if (!iter) [[unlikely]] { + return nullptr; + } + if (iter < end && expFracTable[*iter]) [[unlikely]] { + return finishParse(value, iter, end); + } + return iter; + } + + JSONIFIER_INLINE static unrestrict_t parseInt(value_type& value, read_buffer_ptr iter, read_buffer_ptr end) noexcept { + if (iter < end) [[likely]] { + read_buffer_ptr resultPtr = parseInteger(value, std::bit_cast(iter), std::bit_cast(end)); + if (resultPtr) [[likely]] { + iter += resultPtr - std::bit_cast(iter); + return iter; + } else { + value = 0; + return nullptr; + } + } else { + value = 0; + return nullptr; + } + } + + inline static unrestrict_t finishParse(value_type& value, read_buffer_ptr iter, read_buffer_ptr end) noexcept { + if (peekByte(iter, end) == uint_c_v<'.'>) [[unlikely]] { + ++iter; + return parseFraction(value, iter, end); + } else if (expTable[peekByte(iter, end)]) { + ++iter; + int8_t expSign = 1; + if (peekByte(iter, end) == uint_c_v<'-'>) { + expSign = -1; + ++iter; + } else if (peekByte(iter, end) == uint_c_v<'+'>) { + ++iter; + } + return parseExponent(value, iter, end, expSign); + } + if (!expFracTable[peekByte(iter, end)]) [[likely]] { + return iter; + } else { + return nullptr; + } + } + + JSONIFIER_INLINE static value_type umul128Generic(value_type ab, value_type cd, value_type& hi) noexcept { + value_type aHigh = ab >> 32; + value_type aLow = ab & 0xFFFFFFFF; + value_type bHigh = cd >> 32; + value_type bLow = cd & 0xFFFFFFFF; + value_type loLo = aLow * bLow; + value_type loHi = aLow * bHigh; + value_type hiLo = aHigh * bLow; + value_type hiHi = aHigh * bHigh; + value_type cross = (loLo >> 32) + (loHi & 0xFFFFFFFF) + (hiLo & 0xFFFFFFFF); + value_type lo = (cross << 32) | (loLo & 0xFFFFFFFF); + hi = hiHi + (loHi >> 32) + (hiLo >> 32) + (cross >> 32); + return lo; + } + + JSONIFIER_INLINE static bool multiply(value_type& value, value_type expValue) noexcept { + #if JSONIFIER_HAS_INT128 + const __uint128_t res = static_cast<__uint128_t>(value) * static_cast<__uint128_t>(expValue); + value = static_cast(res); + return res <= std::numeric_limits::max(); + #elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_X64 + value_type values; + value = _umul128(value, expValue, &values); + return values == 0; + #elif JSONIFIER_COMPILER_CLANG || JSONIFIER_COMPILER_GCC + return !__builtin_mul_overflow(value, expValue, &value); + #else + value_type values; + value = umul128Generic(value, expValue, values); + return values == 0; + #endif + } + + JSONIFIER_INLINE static bool divide(value_type& value, value_type expValue) noexcept { + #if JSONIFIER_HAS_INT128 + const __uint128_t dividend = static_cast<__uint128_t>(value); + value = static_cast(dividend / static_cast<__uint128_t>(expValue)); + return (dividend % static_cast<__uint128_t>(expValue)) == 0; + #elif JSONIFIER_COMPILER_MSVC && JSONIFIER_ARCH_X64 + value_type values; + value = _udiv128(0, value, expValue, &values); + return values == 0; + #else + value_type values; + values = value % expValue; + value = value / expValue; + return values == 0; + #endif + } +}; + +#endif +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/string.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/string.hpp new file mode 100644 index 000000000000..cb9f902f38af --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/string.hpp @@ -0,0 +1,1014 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/string.hpp + */ +#pragma once + +#include +#include +#include +#include + +namespace jsonifier { + + class string_view_base; + + template class string_base : internal::alloc_wrapper { + public: + using value_type = char; + using pointer = value_type*; + using const_pointer = const value_type*; + using reference = value_type&; + using const_reference = const value_type&; + using difference_type = std::ptrdiff_t; + using iterator = internal::basic_iterator; + using const_iterator = internal::basic_iterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + using size_type = uint64_t; + using allocator = internal::alloc_wrapper; + using traits_type = std::char_traits; + static constexpr size_type length{ newerSize > 0 ? newerSize - 1 : 0 }; + static constexpr size_type bufferSize = 16 / sizeof(value_type) < 1 ? 1 : 16 / sizeof(value_type); + + static constexpr size_type npos{ std::numeric_limits::max() }; + + JSONIFIER_INLINE static constexpr size_type fromStdPosition(size_t position) noexcept { + return position == std::basic_string_view::npos ? npos : static_cast(position); + } + + JSONIFIER_INLINE string_base() noexcept : internal::alloc_wrapper{}, capacityVal{}, sizeVal{}, dataVal{} { + if constexpr (newerSize > 0) { + reserve(newerSize); + } + } + + template JSONIFIER_INLINE string_base(value_type_newer other) noexcept + : internal::alloc_wrapper{}, capacityVal{}, sizeVal{}, dataVal{} { + if (other) { + const auto newSize = std::char_traits>>::length(other) * + (sizeof(internal::remove_pointer_t) / sizeof(value_type)); + if (newSize > 0 && newSize < maxSize()) [[likely]] { + reserve(newSize); + sizeVal = newSize; + std::uninitialized_copy(other, other + newSize, dataVal); + allocator::construct(&(*this)[newSize], value_type{}); + } + } + } + + template JSONIFIER_INLINE string_base(value_type_newer&& other) noexcept + : internal::alloc_wrapper{}, capacityVal{}, sizeVal{}, dataVal{} { + size_type newSize = other.size() * (sizeof(typename internal::base_t::value_type) / sizeof(value_type)); + if (newSize > 0 && newSize < maxSize()) [[likely]] { + reserve(newSize); + sizeVal = newSize; + std::uninitialized_copy(other.data(), other.data() + newSize, dataVal); + allocator::construct(&(*this)[newSize], value_type{}); + } + } + + JSONIFIER_INLINE string_base(const_iterator other, uint64_t newSize) noexcept : internal::alloc_wrapper{}, capacityVal{}, sizeVal{}, dataVal{} { + if (newSize > 0 && newSize < maxSize()) [[likely]] { + reserve(newSize); + sizeVal = newSize; + std::uninitialized_copy(other.operator->(), other.operator->() + newSize, dataVal); + allocator::construct(&(*this)[newSize], value_type{}); + } + } + + JSONIFIER_INLINE string_base(const_pointer other, uint64_t newSize) noexcept : internal::alloc_wrapper{}, capacityVal{}, sizeVal{}, dataVal{} { + if (newSize > 0 && newSize < maxSize()) [[likely]] { + reserve(newSize); + sizeVal = newSize; + std::uninitialized_copy(other, other + newSize, dataVal); + allocator::construct(&(*this)[newSize], value_type{}); + } + } + + template JSONIFIER_INLINE string_base(value_type_newer other) noexcept + : internal::alloc_wrapper{}, capacityVal{}, sizeVal{}, dataVal{} { + *this = other; + } + + template + requires(size == newerSize && size > 0) + JSONIFIER_INLINE constexpr string_base(const char (&str)[size]) noexcept { + resize(newerSize); + for (uint64_t x = 0; x < length; ++x) { + dataVal[x] = str[x]; + } + dataVal[length] = '\0'; + } + + JSONIFIER_INLINE string_base& operator=(string_base&& other) noexcept { + if (this != &other) [[likely]] { + string_base newValue{ other }; + swap(newValue); + } + return *this; + } + + JSONIFIER_INLINE explicit string_base(string_base&& other) noexcept : internal::alloc_wrapper{}, capacityVal{}, sizeVal{}, dataVal{} { + swap(other); + } + + JSONIFIER_INLINE string_base& operator=(const string_base& other) noexcept { + if (this != &other) [[likely]] { + string_base newValue{ other }; + swap(newValue); + } + return *this; + } + + JSONIFIER_INLINE string_base(const string_base& other) noexcept : internal::alloc_wrapper{}, capacityVal{}, sizeVal{}, dataVal{} { + size_type newSize = other.size(); + if (newSize > 0 && newSize < maxSize()) [[likely]] { + reserve(newSize); + sizeVal = newSize; + std::uninitialized_copy(other.data(), other.data() + newSize, dataVal); + allocator::construct(&(*this)[newSize], value_type{}); + } + } + + template JSONIFIER_INLINE friend string_base operator+=(const value_type_newer (&lhs)[size], const string_base& rhs) noexcept { + string_base newLhs{ lhs }; + newLhs += rhs; + return newLhs; + } + + template JSONIFIER_INLINE friend string_base operator+(const value_type_newer (&lhs)[size], const string_base& rhs) noexcept { + string_base newLhs{ lhs }; + newLhs += rhs; + return newLhs; + } + + template JSONIFIER_INLINE void insert(iterator_type01 where, iterator_type02 start, iterator_type02 end) noexcept { + const int64_t rawNewSize = end - start; + + if (rawNewSize <= 0) [[unlikely]] { + return; + } + + const auto newSize = static_cast(rawNewSize); + const auto posNew = static_cast(where.operator->() - dataVal); + + if (sizeVal + newSize >= capacityVal) [[unlikely]] { + reserve(sizeVal + newSize); + } + + internal::memmoveWrapper(dataVal + posNew + newSize, dataVal + posNew, (sizeVal - posNew) * sizeof(value_type)); + std::copy(start.operator->(), start.operator->() + newSize, dataVal + posNew); + sizeVal += newSize; + allocator::construct(&dataVal[sizeVal], value_type{}); + } + + template JSONIFIER_INLINE friend string_base operator+=(string_type_new&& lhs, const string_base& rhs) noexcept { + string_base newLhs{ lhs }; + newLhs += rhs; + return newLhs; + } + + template JSONIFIER_INLINE friend bool operator==(const string_base& lhs, const value_type_newer& rhs) noexcept { + if (lhs.size() == rhs.size()) { + if (lhs.size() > 0) { + return internal::comparison::compare(lhs.data(), rhs.data(), rhs.size()); + } + return true; + } else { + return false; + } + } + + template JSONIFIER_INLINE friend string_base operator+(string_type_new&& lhs, const string_base& rhs) noexcept { + string_base newLhs{ lhs }; + newLhs += rhs; + return newLhs; + } + + template JSONIFIER_INLINE string_base operator+(const value_type_newer (&rhs)[size]) const noexcept { + string_base newLhs{ *this }; + newLhs += rhs; + return newLhs; + } + + template JSONIFIER_INLINE string_base& operator+=(const value_type_newer (&rhs)[size]) noexcept { + string_base newRhs{ rhs }; + *this += newRhs; + return *this; + } + + template JSONIFIER_INLINE string_base operator+(const string_type_new& rhs) const noexcept { + string_base newLhs{ *this }; + newLhs += rhs; + return newLhs; + } + + template JSONIFIER_INLINE string_base& operator+=(const string_type_new& rhs) noexcept { + append(static_cast(rhs)); + return *this; + } + + template JSONIFIER_INLINE string_base& operator=(value_type_newer&& other) noexcept { + string_base newValue{ other }; + swap(newValue); + return *this; + } + + template JSONIFIER_INLINE string_base& operator=(value_type_newer other) noexcept { + string_base newValue{ std::forward(other) }; + swap(newValue); + return *this; + } + + template JSONIFIER_INLINE string_base& operator+=(string_type_new&& rhs) noexcept { + string_base newRhs{ rhs }; + *this += newRhs; + return *this; + } + + template JSONIFIER_INLINE friend bool operator==(const string_base& lhs, const char (&rhs)[size]) noexcept { + auto rhsLength = traits_type::length(rhs); + return rhsLength == lhs.size() && internal::comparison::compare(lhs.data(), rhs, rhsLength); + } + + template JSONIFIER_INLINE string_base operator+(string_type_new&& rhs) noexcept { + string_base newLhs{ *this }; + newLhs += rhs; + return newLhs; + } + + template JSONIFIER_INLINE string_base& operator=(value_type_newer other) noexcept { + emplace_back(static_cast(other)); + return *this; + } + + JSONIFIER_INLINE string_base substr(size_type position, size_type count = std::numeric_limits::max()) const { + if (static_cast(position) >= static_cast(sizeVal)) [[unlikely]] { + JSONIFIER_THROW(std::out_of_range("Substring position is out of range.")); + } + + count = internal::min(count, sizeVal - position); + + string_base result{}; + if (count > 0) [[likely]] { + result.resize(count); + std::copy(dataVal + position, dataVal + position + count, result.dataVal); + } + return result; + } + + template JSONIFIER_INLINE void append(value_type_newer* values, uint64_t newSize) noexcept { + if (sizeVal + newSize >= capacityVal) [[unlikely]] { + reserve(sizeVal + newSize); + } + if (newSize > 0 && values) [[likely]] { + std::copy(values, values + newSize, dataVal + sizeVal); + sizeVal += newSize; + allocator::construct(&dataVal[sizeVal], value_type{}); + } + } + + template JSONIFIER_INLINE explicit operator std::basic_string() const { + std::basic_string returnValue{}; + if (sizeVal > 0) [[likely]] { + returnValue.resize(sizeVal); + jsonifierMemcpy(returnValue.data(), data(), returnValue.size()); + } + return returnValue; + } + + template JSONIFIER_INLINE size_type findFirstNotOf(arg_types&&... args) const noexcept { + return fromStdPosition(operator std::basic_string_view().find_first_not_of(std::forward(args)...)); + } + + template JSONIFIER_INLINE size_type findLastNotOf(arg_types&&... args) const noexcept { + return fromStdPosition(operator std::basic_string_view().find_last_not_of(std::forward(args)...)); + } + + static consteval size_type maxSize() noexcept { + const size_type allocMax = allocator::maxSize(); + const size_type storageMax = internal::max(allocMax, static_cast(bufferSize)); + return internal::min(static_cast((std::numeric_limits::max)()), storageMax - 1); + } + + template JSONIFIER_INLINE size_type findFirstOf(arg_types&&... args) const noexcept { + return fromStdPosition(operator std::basic_string_view().find_first_of(std::forward(args)...)); + } + + template JSONIFIER_INLINE size_type findLastOf(arg_types&&... args) const noexcept { + return fromStdPosition(operator std::basic_string_view().find_last_of(std::forward(args)...)); + } + + JSONIFIER_INLINE void insert(iterator values, value_type toInsert) noexcept { + const auto positionNew = static_cast(values - begin()); + if (sizeVal + 1 >= capacityVal) [[unlikely]] { + reserve((sizeVal + 1) * 2); + } + const auto newSize = sizeVal - positionNew; + internal::memmoveWrapper(dataVal + positionNew + 1, dataVal + positionNew, newSize * sizeof(value_type)); + allocator::construct(&dataVal[positionNew], toInsert); + ++sizeVal; + } + + template JSONIFIER_INLINE size_type rfind(arg_types&&... args) const noexcept { + return fromStdPosition(operator std::basic_string_view().rfind(std::forward(args)...)); + } + + template JSONIFIER_INLINE size_type find(arg_types&&... args) const noexcept { + return fromStdPosition(operator std::basic_string_view().find(std::forward(args)...)); + } + + template JSONIFIER_INLINE void swap(string_base_new&& other) noexcept { + internal::swap(capacityVal, other.capacityVal); + internal::swap(sizeVal, other.sizeVal); + internal::swap(dataVal, other.dataVal); + } + + JSONIFIER_INLINE const_reference at(size_type index) const { + if (index >= sizeVal) [[unlikely]] { + JSONIFIER_THROW(std::runtime_error{ "Sorry, but that index is beyond the end of this string." }); + } + return dataVal[index]; + } + + JSONIFIER_INLINE reference at(size_type index) { + if (index >= sizeVal) [[unlikely]] { + JSONIFIER_THROW(std::runtime_error{ "Sorry, but that index is beyond the end of this string." }); + } + return dataVal[index]; + } + + JSONIFIER_INLINE operator std::basic_string_view() const noexcept { + return { dataVal, static_cast(sizeVal) }; + } + + JSONIFIER_INLINE void append(const string_base& newSize) noexcept { + if (sizeVal + newSize.size() >= capacityVal) [[unlikely]] { + reserve(sizeVal + newSize.size()); + } + if (newSize.size() > 0) [[likely]] { + std::copy(newSize.data(), newSize.data() + newSize.size(), dataVal + sizeVal); + sizeVal += newSize.size(); + allocator::construct(&dataVal[sizeVal], value_type{}); + } + } + + JSONIFIER_INLINE void resize(size_type newSize) { + if (static_cast(newSize) > 0) [[likely]] { + if (newSize > capacityVal) [[likely]] { + pointer newPtr = allocator::allocate(newSize + 1); + if (dataVal) [[likely]] { + if (sizeVal > 0) [[likely]] { + std::uninitialized_move(dataVal, dataVal + sizeVal, newPtr); + } + allocator::deallocate(dataVal, capacityVal + 1); + } + capacityVal = newSize; + dataVal = newPtr; + std::memset(dataVal + sizeVal, 0, (newSize - sizeVal) * sizeof(value_type)); + allocator::construct(newPtr + newSize, value_type{}); + sizeVal = newSize; + } else if (newSize > sizeVal) [[likely]] { + std::memset(dataVal + sizeVal, 0, (newSize - sizeVal) * sizeof(value_type)); + allocator::construct(dataVal + newSize, value_type{}); + sizeVal = newSize; + } else if (newSize < sizeVal) [[likely]] { + std::destroy(dataVal + newSize, dataVal + sizeVal); + allocator::construct(dataVal + newSize, value_type{}); + sizeVal = newSize; + } + } else { + std::destroy(dataVal, dataVal + sizeVal); + sizeVal = 0; + } + } + + JSONIFIER_INLINE const_reference operator[](size_type index) const noexcept { + return dataVal[index]; + } + + JSONIFIER_INLINE constexpr const_reverse_iterator rbegin() const noexcept { + return const_reverse_iterator{ end() }; + } + + JSONIFIER_INLINE string_base& operator+=(const value_type& rhs) noexcept { + emplace_back(rhs); + return *this; + } + + JSONIFIER_INLINE constexpr const_reverse_iterator rend() const noexcept { + return const_reverse_iterator{ begin() }; + } + + JSONIFIER_INLINE void erase(iterator count) noexcept { + int64_t rawNewSize = count.operator->() - dataVal; + if (rawNewSize == 0) [[unlikely]] { + return; + } else if (rawNewSize > static_cast(sizeVal)) [[unlikely]] { + rawNewSize = static_cast(sizeVal); + } else if (rawNewSize < 0) [[unlikely]] { + return; + } + const auto newSize = static_cast(rawNewSize); + traits_type::move(dataVal, dataVal + newSize, sizeVal - newSize); + sizeVal -= newSize; + allocator::construct(&dataVal[sizeVal], static_cast(0x00u)); + } + + JSONIFIER_INLINE void erase(size_type count) noexcept { + if (count == 0) [[unlikely]] { + return; + } else if (count > sizeVal) [[likely]] { + count = sizeVal; + } + traits_type::move(dataVal, dataVal + count, sizeVal - count); + sizeVal -= count; + allocator::construct(&dataVal[sizeVal], static_cast(0x00u)); + } + + JSONIFIER_INLINE string_base operator+(const value_type& rhs) noexcept { + string_base newLhs{ *this }; + newLhs.emplace_back(rhs); + return newLhs; + } + + JSONIFIER_INLINE constexpr const_iterator begin() const noexcept { + return const_iterator{ dataVal }; + } + + JSONIFIER_INLINE reference operator[](size_type index) noexcept { + return dataVal[index]; + } + + JSONIFIER_INLINE constexpr const_iterator end() const noexcept { + return const_iterator{ dataVal + sizeVal }; + } + + JSONIFIER_INLINE constexpr size_type capacity() const noexcept { + return capacityVal; + } + + JSONIFIER_INLINE void emplace_back(value_type value) noexcept { + if (sizeVal + 1 >= capacityVal) [[unlikely]] { + reserve((sizeVal + 2) * 4); + } + allocator::construct(&dataVal[sizeVal++], value); + allocator::construct(&dataVal[sizeVal], value_type{}); + } + + JSONIFIER_INLINE constexpr reverse_iterator rbegin() noexcept { + return reverse_iterator{ end() }; + } + + JSONIFIER_INLINE constexpr reverse_iterator rend() noexcept { + return reverse_iterator{ begin() }; + } + + JSONIFIER_INLINE void reserve(size_type capacityNew) { + if (capacityNew > capacityVal) [[likely]] { + pointer newPtr = allocator::allocate(capacityNew + 1); + if (dataVal) [[likely]] { + if (sizeVal > 0) [[likely]] { + std::uninitialized_move(dataVal, dataVal + sizeVal, newPtr); + } + allocator::deallocate(dataVal, capacityVal + 1); + } + capacityVal = capacityNew; + dataVal = newPtr; + if (dataVal) { + allocator::construct(&dataVal[sizeVal], value_type{}); + } + } + } + + JSONIFIER_INLINE constexpr size_type size() const noexcept { + return sizeVal; + } + + JSONIFIER_INLINE constexpr bool empty() const noexcept { + return sizeVal == 0; + } + + JSONIFIER_INLINE constexpr iterator begin() noexcept { + return iterator{ dataVal }; + } + + JSONIFIER_INLINE const_pointer data() const noexcept { + return dataVal; + } + + JSONIFIER_INLINE constexpr iterator end() noexcept { + return iterator{ dataVal + sizeVal }; + } + + JSONIFIER_INLINE void clear() noexcept { + if (sizeVal > 0) [[likely]] { + allocator::construct(dataVal, value_type{}); + } + sizeVal = 0; + } + + JSONIFIER_INLINE pointer data() noexcept { + return dataVal; + } + + JSONIFIER_INLINE ~string_base() noexcept { + reset(); + } + + protected: + JSONIFIER_INLINE void reset() noexcept { + if (dataVal && capacityVal) [[likely]] { + if (sizeVal) [[likely]] { + std::destroy(dataVal, dataVal + sizeVal); + sizeVal = 0; + } + allocator::deallocate(dataVal, capacityVal + 1); + dataVal = nullptr; + capacityVal = 0; + } + } + + size_type capacityVal{}; + size_type sizeVal{}; + pointer dataVal{}; + }; + + template + requires(newerSize > 0 && newerSize <= 16) + class string_base { + public: + using value_type = char; + using pointer = value_type*; + using const_pointer = const value_type*; + using reference = value_type&; + using const_reference = const value_type&; + using difference_type = std::ptrdiff_t; + using iterator = internal::basic_iterator; + using const_iterator = internal::basic_iterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + using size_type = uint64_t; + using traits_type = std::char_traits; + + static constexpr size_type length{ newerSize > 0 ? newerSize - 1 : 0 }; + static constexpr size_type bufferSize = 16 / sizeof(value_type) < 1 ? 1 : 16 / sizeof(value_type); + static constexpr size_type npos{ std::numeric_limits::max() }; + + JSONIFIER_INLINE static constexpr size_type fromStdPosition(size_t position) noexcept { + return position == std::basic_string_view::npos ? npos : static_cast(position); + } + + JSONIFIER_INLINE constexpr string_base() noexcept : sizeVal{}, dataVal{} { + if constexpr (newerSize > 0) { + resize(newerSize); + } + } + + JSONIFIER_INLINE constexpr string_base(const char (&str)[newerSize]) noexcept : sizeVal{ length }, dataVal{} { + for (uint64_t x = 0; x < length; ++x) { + dataVal[x] = str[x]; + } + dataVal[length] = value_type{}; + } + + template JSONIFIER_INLINE constexpr string_base(value_type_newer&& other) noexcept : sizeVal{}, dataVal{} { + size_type newSize = other.size() * (sizeof(typename internal::base_t::value_type) / sizeof(value_type)); + if (newSize > 0 && newSize < maxSize()) [[likely]] { + sizeVal = newSize; + std::copy(other.data(), other.data() + newSize, dataVal); + dataVal[newSize] = value_type{}; + } + } + + template JSONIFIER_INLINE constexpr string_base(value_type_newer other) noexcept : sizeVal{}, dataVal{} { + if (other) { + const auto newSize = std::char_traits>>::length(other) * + (sizeof(internal::remove_pointer_t) / sizeof(value_type)); + if (newSize > 0 && newSize < maxSize()) [[likely]] { + sizeVal = newSize; + std::copy(other, other + newSize, dataVal); + dataVal[newSize] = value_type{}; + } + } + } + + template JSONIFIER_INLINE constexpr string_base(value_type_newer other) noexcept : sizeVal{}, dataVal{} { + *this = other; + } + + JSONIFIER_INLINE constexpr string_base(const_pointer other, uint64_t newSize) noexcept : sizeVal{}, dataVal{} { + if (newSize > 0 && newSize < maxSize()) [[likely]] { + sizeVal = newSize; + std::copy(other, other + newSize, dataVal); + dataVal[newSize] = value_type{}; + } + } + + JSONIFIER_INLINE constexpr string_base(const_iterator other, uint64_t newSize) noexcept : sizeVal{}, dataVal{} { + if (newSize > 0 && newSize < maxSize()) [[likely]] { + sizeVal = newSize; + std::copy(other.operator->(), other.operator->() + newSize, dataVal); + dataVal[newSize] = value_type{}; + } + } + + JSONIFIER_INLINE constexpr string_base& operator=(string_base&& other) noexcept { + if (this != &other) [[likely]] { + string_base newValue{ other }; + swap(newValue); + } + return *this; + } + + JSONIFIER_INLINE constexpr explicit string_base(string_base&& other) noexcept : sizeVal{}, dataVal{} { + swap(other); + } + + JSONIFIER_INLINE constexpr string_base& operator=(const string_base& other) noexcept { + if (this != &other) [[likely]] { + string_base newValue{ other }; + swap(newValue); + } + return *this; + } + + JSONIFIER_INLINE constexpr string_base(const string_base& other) noexcept : sizeVal{}, dataVal{} { + size_type newSize = other.size(); + if (newSize > 0 && newSize < maxSize()) [[likely]] { + sizeVal = newSize; + for (size_type x = 0; x < newSize; ++x) { + dataVal[x] = other.dataVal[x]; + } + dataVal[newSize] = value_type{}; + } + } + + template JSONIFIER_INLINE constexpr friend string_base operator+=(string_type_new&& lhs, const string_base& rhs) noexcept { + string_base newLhs{ lhs }; + newLhs += rhs; + return newLhs; + } + + template JSONIFIER_INLINE constexpr friend bool operator==(const string_base& lhs, const value_type_newer& rhs) noexcept { + if (lhs.size() == rhs.size()) { + if (lhs.size() > 0) { + return internal::comparison::compare(lhs.data(), rhs.data(), rhs.size()); + } + return true; + } else { + return false; + } + } + + template JSONIFIER_INLINE constexpr friend string_base operator+(string_type_new&& lhs, const string_base& rhs) noexcept { + string_base newLhs{ lhs }; + newLhs += rhs; + return newLhs; + } + + template JSONIFIER_INLINE constexpr string_base operator+(const value_type_newer (&rhs)[size]) const noexcept { + string_base newLhs{ *this }; + newLhs += rhs; + return newLhs; + } + + template JSONIFIER_INLINE constexpr string_base& operator+=(const value_type_newer (&rhs)[size]) noexcept { + string_base newRhs{ rhs }; + *this += newRhs; + return *this; + } + + template JSONIFIER_INLINE constexpr string_base operator+(const string_type_new& rhs) const noexcept { + string_base newLhs{ *this }; + newLhs += rhs; + return newLhs; + } + + template JSONIFIER_INLINE constexpr string_base& operator+=(const string_type_new& rhs) noexcept { + append(static_cast(rhs)); + return *this; + } + + template JSONIFIER_INLINE constexpr string_base& operator=(value_type_newer&& other) noexcept { + string_base newValue{ other }; + swap(newValue); + return *this; + } + + template JSONIFIER_INLINE constexpr string_base& operator=(value_type_newer other) noexcept { + string_base newValue{ std::forward(other) }; + swap(newValue); + return *this; + } + + template JSONIFIER_INLINE constexpr string_base& operator+=(string_type_new&& rhs) noexcept { + string_base newRhs{ rhs }; + *this += newRhs; + return *this; + } + + template JSONIFIER_INLINE constexpr friend bool operator==(const string_base& lhs, const char (&rhs)[size]) noexcept { + auto rhsLength = traits_type::length(rhs); + return rhsLength == lhs.size() && internal::comparison::compare(lhs.data(), rhs, rhsLength); + } + + template JSONIFIER_INLINE constexpr string_base& operator=(value_type_newer other) noexcept { + emplace_back(static_cast(other)); + return *this; + } + + template JSONIFIER_INLINE constexpr string_base operator+(string_type_new&& rhs) noexcept { + string_base newLhs{ *this }; + newLhs += rhs; + return newLhs; + } + + JSONIFIER_INLINE constexpr string_base substr(size_type position, size_type count = std::numeric_limits::max()) const { + if (static_cast(position) >= static_cast(sizeVal)) [[unlikely]] { + JSONIFIER_THROW(std::out_of_range("Substring position is out of range.")); + } + + count = internal::min(count, sizeVal - position); + + string_base result{}; + if (count > 0) [[likely]] { + result.sizeVal = count; + std::copy(dataVal + position, dataVal + position + count, result.dataVal); + result.dataVal[count] = value_type{}; + } + return result; + } + + template JSONIFIER_INLINE constexpr void append(value_type_newer* values, uint64_t newSize) noexcept { + if (newSize > 0 && values && sizeVal + newSize < maxSize()) [[likely]] { + std::copy(values, values + newSize, dataVal + sizeVal); + sizeVal += newSize; + dataVal[sizeVal] = value_type{}; + } + } + + template + JSONIFIER_INLINE constexpr friend string_base operator+=(const value_type_newer (&lhs)[size], const string_base& rhs) noexcept { + string_base newLhs{ lhs }; + newLhs += rhs; + return newLhs; + } + + template + JSONIFIER_INLINE constexpr friend string_base operator+(const value_type_newer (&lhs)[size], const string_base& rhs) noexcept { + string_base newLhs{ lhs }; + newLhs += rhs; + return newLhs; + } + + template JSONIFIER_INLINE explicit constexpr operator std::basic_string() const { + std::basic_string returnValue{}; + if (sizeVal > 0) [[likely]] { + returnValue.resize(sizeVal); + jsonifierMemcpy(returnValue.data(), data(), returnValue.size()); + } + return returnValue; + } + + template JSONIFIER_INLINE constexpr size_type findFirstNotOf(arg_types&&... args) const noexcept { + return fromStdPosition(operator std::basic_string_view().find_first_not_of(std::forward(args)...)); + } + + template JSONIFIER_INLINE constexpr size_type findLastNotOf(arg_types&&... args) const noexcept { + return fromStdPosition(operator std::basic_string_view().find_last_not_of(std::forward(args)...)); + } + + template JSONIFIER_INLINE constexpr size_type findFirstOf(arg_types&&... args) const noexcept { + return fromStdPosition(operator std::basic_string_view().find_first_of(std::forward(args)...)); + } + + template JSONIFIER_INLINE constexpr size_type findLastOf(arg_types&&... args) const noexcept { + return fromStdPosition(operator std::basic_string_view().find_last_of(std::forward(args)...)); + } + + template + JSONIFIER_INLINE constexpr void insert(iterator_type01 where, iterator_type02 start, iterator_type02 end) noexcept { + const int64_t rawNewSize = end - start; + + if (rawNewSize <= 0) [[unlikely]] { + return; + } + + const auto newSize = static_cast(rawNewSize); + const auto posNew = static_cast(where.operator->() - dataVal); + + std::copy_backward(dataVal + posNew, dataVal + sizeVal, dataVal + sizeVal + newSize); + std::copy(start.operator->(), start.operator->() + newSize, dataVal + posNew); + sizeVal += newSize; + dataVal[sizeVal] = value_type{}; + } + + template JSONIFIER_INLINE constexpr size_type rfind(arg_types&&... args) const noexcept { + return fromStdPosition(operator std::basic_string_view().rfind(std::forward(args)...)); + } + + template JSONIFIER_INLINE constexpr size_type find(arg_types&&... args) const noexcept { + return fromStdPosition(operator std::basic_string_view().find(std::forward(args)...)); + } + + template JSONIFIER_INLINE constexpr void swap(string_base_new&& other) noexcept { + for (size_type x = 0; x < bufferSize; ++x) { + const value_type temp = dataVal[x]; + dataVal[x] = other.dataVal[x]; + other.dataVal[x] = temp; + } + const size_type tempSize = sizeVal; + sizeVal = other.sizeVal; + other.sizeVal = tempSize; + } + + JSONIFIER_INLINE constexpr operator std::basic_string_view() const noexcept { + return { dataVal, static_cast(sizeVal) }; + } + + JSONIFIER_INLINE constexpr void insert(iterator values, value_type toInsert) noexcept { + const auto positionNew = static_cast(values - begin()); + if (sizeVal + 1 >= maxSize()) [[unlikely]] { + return; + } + std::copy_backward(dataVal + positionNew, dataVal + sizeVal, dataVal + sizeVal + 1); + dataVal[positionNew] = toInsert; + ++sizeVal; + dataVal[sizeVal] = value_type{}; + } + + JSONIFIER_INLINE constexpr const_reference operator[](size_type index) const noexcept { + return dataVal[index]; + } + + JSONIFIER_INLINE constexpr const_reference at(size_type index) const { + if (index >= sizeVal) [[unlikely]] { + JSONIFIER_THROW(std::runtime_error{ "Sorry, but that index is beyond the end of this string." }); + } + return dataVal[index]; + } + + JSONIFIER_INLINE constexpr reference at(size_type index) { + if (index >= sizeVal) [[unlikely]] { + JSONIFIER_THROW(std::runtime_error{ "Sorry, but that index is beyond the end of this string." }); + } + return dataVal[index]; + } + + JSONIFIER_INLINE constexpr string_base& operator+=(const value_type& rhs) noexcept { + emplace_back(rhs); + return *this; + } + + JSONIFIER_INLINE constexpr string_base operator+(const value_type& rhs) noexcept { + string_base newLhs{ *this }; + newLhs.emplace_back(rhs); + return newLhs; + } + + JSONIFIER_INLINE constexpr const_reverse_iterator rbegin() const noexcept { + return const_reverse_iterator{ end() }; + } + + JSONIFIER_INLINE constexpr void append(const string_base& other) noexcept { + const size_type newSize = other.size(); + if (newSize > 0 && sizeVal + newSize < maxSize()) [[likely]] { + std::copy(other.data(), other.data() + newSize, dataVal + sizeVal); + sizeVal += newSize; + dataVal[sizeVal] = value_type{}; + } + } + + JSONIFIER_INLINE constexpr reference operator[](size_type index) noexcept { + return dataVal[index]; + } + + JSONIFIER_INLINE constexpr const_reverse_iterator rend() const noexcept { + return const_reverse_iterator{ begin() }; + } + + JSONIFIER_INLINE constexpr void emplace_back(value_type value) noexcept { + if (sizeVal + 1 >= maxSize()) [[unlikely]] { + return; + } + dataVal[sizeVal++] = value; + dataVal[sizeVal] = value_type{}; + } + + JSONIFIER_INLINE constexpr void erase(iterator count) noexcept { + int64_t rawNewSize = count.operator->() - dataVal; + if (rawNewSize == 0) [[unlikely]] { + return; + } else if (rawNewSize > static_cast(sizeVal)) [[unlikely]] { + rawNewSize = static_cast(sizeVal); + } else if (rawNewSize < 0) [[unlikely]] { + return; + } + const auto newSize = static_cast(rawNewSize); + traits_type::move(dataVal, dataVal + newSize, sizeVal - newSize); + sizeVal -= newSize; + dataVal[sizeVal] = static_cast(0x00u); + } + + JSONIFIER_INLINE constexpr void resize(size_type newSize) noexcept { + if (newSize < maxSize()) [[likely]] { + if (newSize > sizeVal) { + for (size_type x = sizeVal; x < newSize; ++x) { + dataVal[x] = value_type{}; + } + } + sizeVal = newSize; + dataVal[sizeVal] = value_type{}; + } + } + + JSONIFIER_INLINE constexpr const_iterator begin() const noexcept { + return const_iterator{ dataVal }; + } + + JSONIFIER_INLINE constexpr void erase(size_type count) noexcept { + if (count == 0) [[unlikely]] { + return; + } else if (count > sizeVal) [[likely]] { + count = sizeVal; + } + traits_type::move(dataVal, dataVal + count, sizeVal - count); + sizeVal -= count; + dataVal[sizeVal] = static_cast(0x00u); + } + + JSONIFIER_INLINE constexpr const_iterator end() const noexcept { + return const_iterator{ dataVal + sizeVal }; + } + + JSONIFIER_INLINE constexpr size_type capacity() const noexcept { + return bufferSize; + } + + JSONIFIER_INLINE constexpr const_pointer data() const noexcept { + return dataVal; + } + + JSONIFIER_INLINE constexpr reverse_iterator rbegin() noexcept { + return reverse_iterator{ end() }; + } + + JSONIFIER_INLINE constexpr reverse_iterator rend() noexcept { + return reverse_iterator{ begin() }; + } + + JSONIFIER_INLINE constexpr size_type size() const noexcept { + return sizeVal; + } + + JSONIFIER_INLINE constexpr bool empty() const noexcept { + return sizeVal == 0; + } + + JSONIFIER_INLINE constexpr iterator begin() noexcept { + return iterator{ dataVal }; + } + + JSONIFIER_INLINE constexpr iterator end() noexcept { + return iterator{ dataVal + sizeVal }; + } + + JSONIFIER_INLINE constexpr pointer data() noexcept { + return dataVal; + } + + JSONIFIER_INLINE constexpr void clear() noexcept { + dataVal[0] = value_type{}; + sizeVal = 0; + } + + static consteval size_type maxSize() noexcept { + return bufferSize; + } + + JSONIFIER_INLINE constexpr ~string_base() noexcept = default; + + protected: + value_type dataVal[bufferSize + 1]{}; + size_type sizeVal{}; + }; + + using string = string_base<17>; + + template string_base(const char (&)[size]) -> string_base; + + template + JSONIFIER_INLINE internal::basic_stream& operator<<(internal::basic_stream& os, const string_base& input) noexcept { + os << std::basic_string_view{ input.data(), input.size() }; + return os; + } + + template JSONIFIER_INLINE std::ostream& operator<<(std::ostream& os, const string_base& input) noexcept { + os << std::basic_string_view{ input.data(), input.size() }; + return os; + } +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/string_literal.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/string_literal.hpp new file mode 100644 index 000000000000..9865339fc5c0 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/string_literal.hpp @@ -0,0 +1,147 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/string_literal.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::internal { + + template struct string_literal { + using value_type = char; + using size_type = uint64_t; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; + static constexpr size_type length{ sizeVal > 0 ? sizeVal - 1 : 0 }; + + JSONIFIER_INLINE constexpr string_literal() noexcept = default; + + JSONIFIER_INLINE constexpr string_literal(const char (&str)[sizeVal]) noexcept { + for (uint64_t x = 0; x < length; ++x) { + values[x] = str[x]; + } + values[length] = '\0'; + } + + template JSONIFIER_INLINE friend constexpr auto operator+(const value_type (&lhs)[sizeNew], const string_literal& str) noexcept { + return string_literal{ lhs } + str; + } + + template JSONIFIER_INLINE constexpr string_literal operator+=(const string_literal& str) const noexcept { + string_literal newLiteral{}; + std::copy(values, values + size(), newLiteral.data()); + std::copy(str.data(), str.data() + sizeNew, newLiteral.data() + size()); + return newLiteral; + } + + template JSONIFIER_INLINE constexpr string_literal operator+(const string_literal& str) const noexcept { + string_literal newLiteral{}; + std::copy(values, values + size(), newLiteral.data()); + std::copy(str.data(), str.data() + sizeNew, newLiteral.data() + size()); + return newLiteral; + } + + template JSONIFIER_INLINE constexpr string_literal operator+=(const value_type (&str)[sizeNew]) const noexcept { + string_literal newLiteral{}; + std::copy(values, values + size(), newLiteral.data()); + std::copy(str, str + sizeNew, newLiteral.data() + size()); + return newLiteral; + } + + template JSONIFIER_INLINE constexpr string_literal operator+(const value_type (&str)[sizeNew]) const noexcept { + string_literal newLiteral{}; + std::copy(values, values + size(), newLiteral.data()); + std::copy(str, str + sizeNew, newLiteral.data() + size()); + return newLiteral; + } + + template JSONIFIER_INLINE constexpr string_literal substr() const noexcept { + static_assert(offsetNew + count <= length); + string_literal returnValue{}; + std::copy(values + offsetNew, values + offsetNew + count, returnValue.values); + returnValue.values[count] = '\0'; + return returnValue; + } + + JSONIFIER_INLINE constexpr const_reference operator[](size_type index) const noexcept JSONIFIER_LIFETIME_BOUND { + return values[index]; + } + + JSONIFIER_INLINE constexpr reference operator[](size_type index) noexcept JSONIFIER_LIFETIME_BOUND { + return values[index]; + } + + template JSONIFIER_INLINE constexpr operator string_type() const noexcept { + string_type returnValues{ values, length }; + return returnValues; + } + + JSONIFIER_INLINE constexpr const_pointer data() const noexcept JSONIFIER_LIFETIME_BOUND { + return values; + } + + JSONIFIER_INLINE constexpr pointer data() noexcept JSONIFIER_LIFETIME_BOUND { + return values; + } + + JSONIFIER_INLINE static constexpr size_type size() noexcept { + return length; + } + + alignas(simdBytesPerRegister) value_type values[sizeVal]{}; + }; + + template string_literal(const char (&)[sizeVal]) -> string_literal; + + template inline basic_stream& operator<<(basic_stream& os, const string_literal& input) noexcept { + os << std::basic_string_view{ input.data(), input.size() }; + return os; + } + + template inline std::ostream& operator<<(std::ostream& os, const string_literal& input) noexcept { + os << std::basic_string_view{ input.data(), input.size() }; + return os; + } + + template consteval uint64_t escapedKeyLength() noexcept { + uint64_t newLength{}; + for (uint64_t x = 0; x < literal.size(); ++x) { + const char c = literal[x]; + newLength += (c == '"' || c == '\\') ? 2 : 1; + } + return newLength; + } + + template consteval string_literal() + 1> escapeKeyLiteral() noexcept { + constexpr uint64_t newLength{ escapedKeyLength() }; + string_literal returnValues{}; + uint64_t outIndex{}; + for (uint64_t x = 0; x < literal.size(); ++x) { + const char c = literal[x]; + if (c == '"' || c == '\\') { + returnValues[outIndex] = '\\'; + ++outIndex; + } + returnValues[outIndex] = c; + ++outIndex; + } + returnValues[newLength] = '\0'; + return returnValues; + } + + template consteval string_literal stringLiteralFromView(string_type str) noexcept { + string_literal sl{}; + std::copy_n(str.data(), str.size(), sl.values); + sl[index] = '\0'; + return sl; + } + + template inline constexpr auto escapedKeyLiteral{ escapeKeyLiteral() }; + +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/string_utils.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/string_utils.hpp new file mode 100644 index 000000000000..2c9b51f741ea --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/string_utils.hpp @@ -0,0 +1,978 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/string_utils.hpp + */ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace jsonifier::internal { + + template alignas(64) inline constexpr uint32_t digitToVal32[]{ 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0x0u, 0x1u, 0x2u, 0x3u, 0x4u, 0x5u, 0x6u, 0x7u, 0x8u, 0x9u, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xAu, 0xBu, 0xCu, 0xDu, 0xEu, 0xFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xAu, 0xBu, 0xCu, 0xDu, 0xEu, 0xFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0x0u, 0x10u, 0x20u, 0x30u, 0x40u, 0x50u, 0x60u, 0x70u, 0x80u, 0x90u, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xA0u, 0xB0u, 0xC0u, 0xD0u, 0xE0u, 0xF0u, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xA0u, 0xB0u, 0xC0u, 0xD0u, 0xE0u, 0xF0u, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0x0u, 0x100u, 0x200u, 0x300u, 0x400u, + 0x500u, 0x600u, 0x700u, 0x800u, 0x900u, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xA00u, 0xB00u, 0xC00u, 0xD00u, 0xE00u, + 0xF00u, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xA00u, 0xB00u, 0xC00u, 0xD00u, 0xE00u, 0xF00u, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0x0u, 0x1000u, 0x2000u, 0x3000u, 0x4000u, 0x5000u, 0x6000u, 0x7000u, 0x8000u, 0x9000u, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xA000u, 0xB000u, 0xC000u, 0xD000u, 0xE000u, 0xF000u, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xA000u, 0xB000u, 0xC000u, 0xD000u, 0xE000u, 0xF000u, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, + 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu }; + + // Sampled from Stephen Berry's library, Glaze: https://github.com/StephenBerry/Glaze + template [[maybe_unused]] JSONIFIER_INLINE void skipStringImpl(basic_iterator01& string1, uint64_t lengthNew) noexcept { + if (static_cast(lengthNew) > 0) { + const auto endIter = string1 + lengthNew; + while (string1 < endIter) { + auto* newIter = char_comparison<'"', base_t>::memchar(string1, lengthNew); + if (newIter) { + string1 = newIter; + lengthNew = static_cast(endIter - string1); + + auto* prev = string1 - 1; + while (*prev == '\\') { + --prev; + } + if (static_cast(string1 - prev) % 2) { + break; + } + ++string1; + lengthNew = static_cast(endIter - string1); + } else { + break; + } + } + } + } + + // Sampled from Dr. Lemire's library, simdjson: https://github.com/simdjson/simdjson + JSONIFIER_INLINE uint32_t hexToU32NoCheck(read_buffer_ptr string1) noexcept { + return digitToVal32<>[630ull + static_cast(string1[0])] | digitToVal32<>[420ull + static_cast(string1[1])] | + digitToVal32<>[210ull + static_cast(string1[2])] | digitToVal32<>[0ull + static_cast(string1[3])]; + } + + // Sampled from Dr. Lemire's library, simdjson: https://github.com/simdjson/simdjson + template JSONIFIER_INLINE uint32_t codePointToUtf8(uint32_t cp, char_type* c) noexcept { + if (cp <= 0x7F) { + c[0] = static_cast(cp); + return 1; + } + if (cp <= 0x7FF) { + c[0] = static_cast(0xC0 | ((cp >> 6) & 0x1F)); + c[1] = static_cast(0x80 | (cp & 0x3F)); + return 2; + } + if (cp <= 0xFFFF) { + c[0] = static_cast(0xE0 | ((cp >> 12) & 0x0F)); + c[1] = static_cast(0x80 | ((cp >> 6) & 0x3F)); + c[2] = static_cast(0x80 | (cp & 0x3F)); + return 3; + } + if (cp <= 0x10FFFF) { + c[0] = static_cast(0xF0 | ((cp >> 18) & 0x07)); + c[1] = static_cast(0x80 | ((cp >> 12) & 0x3F)); + c[2] = static_cast(0x80 | ((cp >> 6) & 0x3F)); + c[3] = static_cast(0x80 | (cp & 0x3F)); + return 4; + } + return 0; + } + + // Sampled from Dr. Lemire's library, simdjson: https://github.com/simdjson/simdjson + template + JSONIFIER_INLINE bool handleUnicodeCodePoint(basic_iterator01& srcPtr, basic_iterator02& dstPtr, basic_iterator01 srcEnd) noexcept { + static constexpr uint8_t bs{ uint_c_v<'\\'> }; + static constexpr uint8_t u{ uint_c_v<'u'> }; + + if ((srcPtr + 6) > srcEnd) [[unlikely]] { + return false; + } + + uint32_t codePoint = hexToU32NoCheck(srcPtr + 2); + srcPtr += 6; + if (codePoint >= 0xD800 && codePoint <= 0xDFFF) { + if (codePoint >= 0xDC00) { + return false; + } + if ((srcPtr + 6) > srcEnd) [[unlikely]] { + return false; + } + if (((srcPtr[0] << 8) | srcPtr[1]) != ((bs << 8) | u)) { + return false; + } + uint32_t lowSurrogate = hexToU32NoCheck(srcPtr + 2); + uint32_t lowBit = lowSurrogate - 0xDC00; + if (lowBit >> 10) { + return false; + } + codePoint = (((codePoint - 0xD800) << 10) | lowBit) + 0x10000; + srcPtr += 6; + } + const uint64_t offset = codePointToUtf8(codePoint, dstPtr); + dstPtr += offset; + return offset > 0; + } + + consteval array genEscapeMap() noexcept { + array returnValues{}; + returnValues[uint_c_v<'"'>] = '\"'; + returnValues[uint_c_v<'\\'>] = '\\'; + returnValues[uint_c_v<'/'>] = '/'; + returnValues[uint_c_v<'b'>] = '\b'; + returnValues[uint_c_v<'f'>] = '\f'; + returnValues[uint_c_v<'n'>] = '\n'; + returnValues[uint_c_v<'r'>] = '\r'; + returnValues[uint_c_v<'t'>] = '\t'; + return returnValues; + } + + alignas(64) inline constexpr auto escapeMapRaw{ genEscapeMap() }; + + inline constexpr const char* __restrict escapeMap{ escapeMapRaw.data() }; + + template + JSONIFIER_INLINE basic_iterator02 unescapeImpl(basic_iterator01 string1Start, const basic_iterator01 string1End, basic_iterator02 string2) noexcept { + char escapeChar; + while (string1Start < string1End) { + escapeChar = *string1Start; + if (escapeChar == '\\') { + escapeChar = string1Start[1]; + if (escapeChar == 'u') { + if (!handleUnicodeCodePoint(string1Start, string2, string1End)) { + return nullptr; + } + continue; + } + escapeChar = escapeMap[static_cast(escapeChar)]; + if (escapeChar == 0) { + return nullptr; + } + *string2 = escapeChar; + ++string2; + string1Start += 2; + continue; + } + *string2 = escapeChar; + ++string2; + ++string1Start; + } + return string2; + } + + template struct string_parse_executor; + + template struct string_parse_executor> { + template JSONIFIER_INLINE static bool impl(arg_types&&... args) noexcept { + return ((executor_type::template impl(std::forward(args)...) != nullptr) && ...); + } + }; + +#if JSONIFIER_AVX512 + inline constexpr uint64_t startIndex = 0; +#elif JSONIFIER_AVX2 + inline constexpr uint64_t startIndex = 1; +#else + inline constexpr uint64_t startIndex = 2; +#endif + + inline constexpr uint64_t listSize = 3; + + template struct make_ascending_range_impl : make_ascending_range_impl {}; + template struct make_ascending_range_impl { + using type = integer_sequence; + }; + + template using make_ascending_range = typename make_ascending_range_impl::type; + + template struct string_scanner { + struct scan_result { + uint64_t outLength{ std::numeric_limits::max() }; + uint64_t rawLength{}; + }; + + static constexpr uint64_t npos{ std::numeric_limits::max() }; + + struct scan_state { + utf8_validation_state validationState{}; + uint64_t rawLength{}; + uint64_t outLength{}; + bool complete{}; + }; + + struct string_scan_step { + template JSONIFIER_INLINE static basic_iterator01 impl(basic_iterator01& string1Start, + const basic_iterator01 string1End, basic_iterator02& string2, const basic_iterator01 stringStart, const basic_iterator02 outStart, scan_state& scanState) noexcept { + using simd_list_local = type_list_element_t; + using integer_type = typename simd_list_local::integer_type; + static constexpr auto avx_type = simd_list_local::type::avx_type; + using simd_type_local = typename simd_type_wrapper::type; + static constexpr uint64_t bytesProcessed = simd_list_local::bytesProcessed; + if (scanState.complete || scanState.outLength == std::numeric_limits::max()) { + return nullptr; + } + utf8_register_validator> validator{ scanState.validationState }; + const simd_type_local simdValues00 = simd::gatherValue('\\'); + const simd_type_local simdValues01 = simd::gatherValue('"'); + const simd_type_local simdValues02 = simd::gatherValue(static_cast(32)); + const auto stringEndNew = string1End - bytesProcessed; + while (string1Start < stringEndNew) { + const auto registerStart = string1Start; + const simd_type_local simdValue = simd::gatherValuesU(string1Start); + simd::storeU(simdValue, string2); + const integer_type delimiters = static_cast(simd::opBitMask( + simd::opOr(simd::opOr(simd::opCmpEqRaw(simdValue, simdValues00), simd::opCmpEqRaw(simdValue, simdValues01)), simd::opCmpLtRaw(simdValue, simdValues02)))); + if (delimiters == static_cast(0)) { + validator.checkRegister(simdValue); + string1Start += bytesProcessed; + string2 += bytesProcessed; + continue; + } + const uint64_t offset = static_cast(simd::countrZeroUnsafe(delimiters)); + const uint8_t foundChar = static_cast(string1Start[offset]); + if (foundChar < 32) [[unlikely]] { + scanState.outLength = std::numeric_limits::max(); + scanState.complete = true; + return nullptr; + } + validator.checkPartial(registerStart, offset); + string1Start += offset; + string2 += offset; + if (validator.errors()) [[unlikely]] { + scanState.outLength = std::numeric_limits::max(); + scanState.complete = true; + return nullptr; + } + if (foundChar == uint_c_v<'"'>) { + scanState.rawLength = static_cast(string1Start - stringStart); + scanState.outLength = static_cast(string2 - outStart); + scanState.complete = true; + return nullptr; + } + if (string1Start + 1 >= string1End) [[unlikely]] { + scanState.outLength = std::numeric_limits::max(); + scanState.complete = true; + return nullptr; + } + if (!handleEscape(string1Start, string1End, string2)) [[unlikely]] { + scanState.outLength = std::numeric_limits::max(); + scanState.complete = true; + return nullptr; + } + validator.reset(); + } + validator.flush(); + + if constexpr (index == 2) { + auto validateFrom = string1Start; + + while (string1Start < string1End) { + const uint8_t currentChar = static_cast(*string1Start); + + if (currentChar == uint_c_v<'"'>) { + validator.checkPartial(validateFrom, static_cast(string1Start - validateFrom)); + if (validator.errors()) [[unlikely]] { + break; + } + validator.flush(); + scanState.rawLength = static_cast(string1Start - stringStart); + scanState.outLength = static_cast(string2 - outStart); + scanState.complete = true; + return nullptr; + } + + if (currentChar == uint_c_v<'\\'>) { + validator.checkPartial(validateFrom, static_cast(string1Start - validateFrom)); + if (validator.errors()) [[unlikely]] { + break; + } + if (string1Start + 1 >= string1End) [[unlikely]] { + break; + } + if (!handleEscape(string1Start, string1End, string2)) [[unlikely]] { + break; + } + validator.reset(); + validateFrom = string1Start; + continue; + } + + if (currentChar < 32) [[unlikely]] { + break; + } + + *string2 = static_cast>(currentChar); + ++string2; + ++string1Start; + } + + validator.flush(); + scanState.outLength = std::numeric_limits::max(); + scanState.complete = true; + } + + return string1Start; + } + }; + + template + JSONIFIER_INLINE static scan_result impl(basic_iterator01 string1Start, const basic_iterator01 string1End, basic_iterator02 string2) noexcept { + const auto stringStart = string1Start; + const auto outStart = string2; + scan_state scanState{}; + + string_parse_executor>::impl(string1Start, string1End, string2, stringStart, outStart, scanState); + + if (scanState.outLength == std::numeric_limits::max()) [[unlikely]] { + return {}; + } + return { scanState.outLength, scanState.rawLength }; + } + + template + JSONIFIER_INLINE static bool handleEscape(basic_iterator01& srcPtr, const basic_iterator01 srcEnd, basic_iterator02& dstPtr) noexcept { + const uint8_t escapeChar = static_cast(srcPtr[1]); + if (escapeChar == uint_c_v<'u'>) { + return handleUnicodeCodePoint(srcPtr, dstPtr, srcEnd); + } + const uint8_t decoded = static_cast(escapeMap[escapeChar]); + if (decoded == 0u) [[unlikely]] { + return false; + } + *dstPtr = static_cast>(decoded); + ++dstPtr; + srcPtr += 2; + return true; + } + }; + + template struct string_prescanner { + struct prescan_result { + uint64_t length{}; + bool nonAscii{}; + bool escaped{}; + bool control{}; + bool found{}; + }; + + struct scan_state { + prescan_result result{}; + bool complete{}; + }; + + struct string_prescan_step { + template JSONIFIER_INLINE static basic_iterator01 impl(basic_iterator01& string1Start, const basic_iterator01 string1End, + const basic_iterator01 stringStart, scan_state& scanState) noexcept { + using simd_list_local = type_list_element_t; + using integer_type = typename simd_list_local::integer_type; + static constexpr auto avx_type = simd_list_local::type::avx_type; + using simd_type_local = typename simd_type_wrapper::type; + static constexpr uint64_t bytesProcessed = simd_list_local::bytesProcessed; + if (scanState.complete) { + return nullptr; + } + const simd_type_local simdValues00 = simd::gatherValue('\\'); + const simd_type_local simdValues01 = simd::gatherValue('"'); + const simd_type_local simdValues02 = simd::gatherValue(static_cast(32)); + const simd_type_local simdValues03 = simd::gatherValue(static_cast(0x80)); + const auto stringEndNew = string1End - bytesProcessed; + while (string1Start < stringEndNew) { + const simd_type_local simdValue = simd::gatherValuesU(string1Start); + const integer_type delimiters = + static_cast(simd::opBitMask(simd::opOr(simd::opCmpEqRaw(simdValue, simdValues00), simd::opCmpEqRaw(simdValue, simdValues01)))); + const integer_type controls = static_cast(simd::opBitMask(simd::opCmpLtRaw(simdValue, simdValues02))); + const integer_type nonAscii = static_cast(~static_cast(simd::opBitMask(simd::opCmpLtRaw(simdValue, simdValues03)))); + if (delimiters == static_cast(0)) { + scanState.result.control |= controls != static_cast(0); + scanState.result.nonAscii |= nonAscii != static_cast(0); + string1Start += bytesProcessed; + continue; + } + const uint64_t offset = static_cast(simd::countrZeroUnsafe(delimiters)); + const integer_type preceding = static_cast((static_cast(1) << offset) - static_cast(1)); + scanState.result.control |= (controls & preceding) != static_cast(0); + scanState.result.nonAscii |= (nonAscii & preceding) != static_cast(0); + string1Start += offset; + if (*string1Start == '"') { + scanState.result.length = static_cast(string1Start - stringStart); + scanState.result.found = true; + scanState.complete = true; + return nullptr; + } + scanState.result.escaped = true; + string1Start += 2; + } + + if constexpr (index == 2) { + scanState.complete = true; + while (string1Start < string1End) { + const uint8_t currentChar = static_cast(*string1Start); + if (currentChar == uint_c_v<'"'>) { + scanState.result.length = static_cast(string1Start - stringStart); + scanState.result.found = true; + return nullptr; + } + if (currentChar == uint_c_v<'\\'>) { + scanState.result.escaped = true; + string1Start += 2; + continue; + } + scanState.result.control |= currentChar < 32; + scanState.result.nonAscii |= currentChar >= 0x80; + ++string1Start; + } + return nullptr; + } else { + return string1Start; + } + } + }; + + template JSONIFIER_INLINE static prescan_result impl(basic_iterator01 string1Start, const basic_iterator01 string1End) noexcept { + const auto stringStart = string1Start; + scan_state scanState{}; + string_parse_executor>::impl(string1Start, string1End, stringStart, scanState); + return scanState.result; + } + }; + + template struct string_validating_copier { + struct copy_state { + utf8_validation_state validationState{}; + bool complete{}; + bool failed{}; + }; + + struct string_copy_step { + template JSONIFIER_INLINE static basic_iterator01 impl(basic_iterator01& string1Start, + const basic_iterator01 string1End, basic_iterator02& string2, copy_state& copyState) noexcept { + using simd_list_local = type_list_element_t; + using integer_type = typename simd_list_local::integer_type; + static constexpr auto avx_type = simd_list_local::type::avx_type; + using simd_type_local = typename simd_type_wrapper::type; + static constexpr uint64_t bytesProcessed = simd_list_local::bytesProcessed; + if (copyState.complete) { + return nullptr; + } + utf8_register_validator> validator{ copyState.validationState }; + const simd_type_local simdValues00 = simd::gatherValue(static_cast(32)); + while (static_cast(string1End - string1Start) >= bytesProcessed) { + const simd_type_local simdValue = simd::gatherValuesU(string1Start); + if (static_cast(simd::opBitMask(simd::opCmpLtRaw(simdValue, simdValues00))) != static_cast(0)) [[unlikely]] { + copyState.failed = true; + copyState.complete = true; + return nullptr; + } + validator.checkRegister(simdValue); + simd::storeU(simdValue, string2); + string1Start += bytesProcessed; + string2 += bytesProcessed; + } + validator.flush(); + + if constexpr (index == 2) { + copyState.complete = true; + const uint64_t remaining = static_cast(string1End - string1Start); + for (uint64_t x = 0; x < remaining; ++x) { + if (static_cast(string1Start[x]) < 32) [[unlikely]] { + copyState.failed = true; + return nullptr; + } + } + validator.checkPartial(string1Start, remaining); + if (validator.errors()) [[unlikely]] { + copyState.failed = true; + return nullptr; + } + jsonifierMemcpy(string2, string1Start, remaining); + return nullptr; + } else { + return string1Start; + } + } + }; + + template + JSONIFIER_INLINE static bool impl(basic_iterator01 string1Start, uint64_t length, basic_iterator02 string2) noexcept { + const auto string1End = string1Start + length; + copy_state copyState{}; + string_parse_executor>::impl(string1Start, string1End, string2, copyState); + return !copyState.failed; + } + }; + + consteval array, 256ULL> genEscapeTable() noexcept { + array, 256ULL> returnValues{}; + constexpr char hex[]{ "0123456789ABCDEF" }; + for (uint64_t x = 0; x < 32ULL; ++x) { + returnValues[x] = { { '\\', 'u', '0', '0', hex[x >> 4], hex[x & 0xF], '\0', 6 } }; + } + auto setShort = [&](char value, char escaped) { + returnValues[static_cast(value)] = { { '\\', escaped, '\0', '\0', '\0', '\0', '\0', 2 } }; + }; + setShort('\b', 'b'); + setShort('\t', 't'); + setShort('\n', 'n'); + setShort('\f', 'f'); + setShort('\r', 'r'); + setShort('"', '"'); + setShort('\\', '\\'); + return returnValues; + } + + alignas(64) inline constexpr auto escapeTable{ genEscapeTable() }; + + template struct string_serializer { + struct string_serialize_step { + template + JSONIFIER_INLINE static auto* impl(basic_iterator01& string1Start, const basic_iterator01 string1End, basic_iterator02& string2) noexcept { + using simd_list_local = type_list_element_t; + using integer_type = typename simd_list_local::integer_type; + using simd_type = typename simd_list_local::type::type; + static constexpr uint64_t bytesProcessed = simd_list_local::bytesProcessed; + static constexpr uint64_t bitsPerByte = sizeof(integer_type) * 8 / bytesProcessed; + static constexpr uint64_t laneCount = bytesProcessed / 16; + static constexpr uint64_t laneBitCount = bitsPerByte * 16; + static constexpr integer_type laneMask = static_cast(std::numeric_limits::max() >> (sizeof(integer_type) * 8 - laneBitCount)); + + const auto stringEndNew = string1End - bytesProcessed; + const simd_type simdValues01 = simd::gatherValue('"'); + const simd_type simdValues02 = simd::gatherValue('\\'); + const simd_type simdValues03 = simd::gatherValue(static_cast(32)); + while (string1Start <= stringEndNew) { + const simd_type simdValue = simd::gatherValuesU(string1Start); + const integer_type escapeMask = escapeBits(simdValue, simdValues01, simdValues02, simdValues03); + if (escapeMask == 0) [[likely]] { + simd::storeU(simdValue, string2); + string2 += bytesProcessed; + string1Start += bytesProcessed; + } else { + if constexpr (laneCount > 0) { + const integer_type laneEscapes = static_cast(static_cast(escapeMask) & laneMask); + string2 = escapeLane(string1Start, string1Start + 16, string2, laneEscapes); + } + if constexpr (laneCount > 1) { + const integer_type laneEscapes = static_cast(static_cast(escapeMask >> (laneBitCount * 1)) & laneMask); + string2 = escapeLane(string1Start, string1Start + 16, string2, laneEscapes); + } + if constexpr (laneCount > 2) { + const integer_type laneEscapes = static_cast(static_cast(escapeMask >> (laneBitCount * 2)) & laneMask); + string2 = escapeLane(string1Start, string1Start + 16, string2, laneEscapes); + } + if constexpr (laneCount > 3) { + const integer_type laneEscapes = static_cast(static_cast(escapeMask >> (laneBitCount * 3)) & laneMask); + string2 = escapeLane(string1Start, string1Start + 16, string2, laneEscapes); + } + } + } + return string1Start; + } + }; + + static constexpr uint64_t pageSize = 4096; + + // Sampled from Dr. Lemire's library, simdjson: https://github.com/simdjson/simdjson + template + JSONIFIER_INLINE static basic_iterator02 overlappedFinish(basic_iterator01& string1Start, const basic_iterator01 string1End, basic_iterator02 string2) noexcept { + const uint64_t remaining = static_cast(string1End - string1Start); + if (remaining == 0) { + return string2; + } + + using simd_list_local = type_list_element_t<2, simd::avx_integer_list>; + using integer_type = typename simd_list_local::integer_type; + using simd_type = typename simd_list_local::type::type; + static constexpr uint64_t bitsPerByte = sizeof(integer_type) * 8 / 16; + const simd_type simdValue = simd::gatherValuesU(string1End - 16); + const simd_type simdValues01 = simd::gatherValue('"'); + const simd_type simdValues02 = simd::gatherValue('\\'); + const simd_type simdValues03 = simd::gatherValue(static_cast(32)); + const integer_type escapeMask = + static_cast(escapeBits(simdValue, simdValues01, simdValues02, simdValues03) >> (bitsPerByte * (16 - remaining))); + if (escapeMask == 0) [[likely]] { + copyUpTo16(string2, string1Start, remaining); + string1Start = string1End; + return string2 + remaining; + } + return escapeLane(string1Start, string1End, string2, escapeMask); + } + + template JSONIFIER_INLINE static basic_iterator02 escapeLane( + basic_iterator01& string1Start, const basic_iterator01 laneEnd, basic_iterator02 string2, integer_type escapeMask) noexcept { + while (escapeMask != 0) { + const uint64_t runLength = simd::postCmpTzcntUnsafe(escapeMask); + copyUpTo16(string2, string1Start, runLength); + string2 += runLength; + string1Start += runLength; + const auto& entry = escapeTable[static_cast(*string1Start)]; + pow2MemcpyWrapper<8>(string2, entry.data()); + string2 += static_cast(entry[7]); + ++string1Start; + escapeMask = static_cast(static_cast(escapeMask >> (bitsPerByte * runLength)) >> bitsPerByte); + } + const uint64_t trailing = static_cast(laneEnd - string1Start); + copyUpTo16(string2, string1Start, trailing); + string1Start = laneEnd; + return string2 + trailing; + } + + template JSONIFIER_INLINE static integer_type escapeBits(const simd_type& simdValue, const simd_type& simdValues01, + const simd_type& simdValues02, const simd_type& simdValues03) noexcept { + const auto flagged = + simd::opOr(simd::opOr(simd::opCmpLtRaw(simdValue, simdValues03), simd::opCmpEqRaw(simdValue, simdValues02)), simd::opCmpEqRaw(simdValue, simdValues01)); + return static_cast(simd::opBitMaskRaw(flagged)); + } + + template + JSONIFIER_INLINE static basic_iterator02 scalarImpl(basic_iterator01& string1Start, const basic_iterator01 string1End, basic_iterator02 string2) noexcept { + for (; string1Start < string1End; ++string1Start) { + const uint8_t nextChar = static_cast(*string1Start); + const auto& entry = escapeTable[nextChar]; + if (const uint8_t escapeSize = static_cast(entry[7]); escapeSize) { + pow2MemcpyWrapper<8>(string2, entry.data()); + string2 += escapeSize; + } else { + *string2 = nextChar; + ++string2; + } + } + return string2; + } + + template + JSONIFIER_INLINE static basic_iterator02 swarFinish(basic_iterator01& string1Start, const basic_iterator01 string1End, basic_iterator02 string2) noexcept { + while (string1End - string1Start >= 8) { + uint64_t word; + pow2MemcpyWrapper<8>(&word, string1Start); + pow2MemcpyWrapper<8>(string2, &word); + if constexpr (std::endian::native == std::endian::big) { + word = internal::byteswap(word); + } + const uint64_t flags = flagMask(word); + if (!flags) { + string1Start += 8; + string2 += 8; + continue; + } + const uint64_t offset = static_cast(simd::countrZeroUnsafe(flags)) >> 3; + string1Start += offset; + string2 += offset; + const auto& entry = escapeTable[static_cast(*string1Start)]; + pow2MemcpyWrapper<8>(string2, entry.data()); + string2 += static_cast(entry[7]); + ++string1Start; + } + const uint64_t remaining = static_cast(string1End - string1Start); + if (remaining == 0) { + return string2; + } + if constexpr (std::endian::native == std::endian::little) { + uint64_t word; + pow2MemcpyWrapper<8>(&word, string1End - 8); + const uint64_t shift = (8 - remaining) * 8; + word >>= shift; + const uint64_t flags = flagMask(word) & (~uint64_t{ 0 } >> shift); + if (!flags) { + pow2MemcpyWrapper<8>(string2, &word); + string1Start = string1End; + return string2 + remaining; + } + } + return scalarImpl(string1Start, string1End, string2); + } + + template + JSONIFIER_INLINE static basic_iterator02 pageSafeShortImpl(basic_iterator01& string1Start, const uint64_t length, basic_iterator02 string2) noexcept { + using simd_list_local = type_list_element_t<2, simd::avx_integer_list>; + using integer_type = typename simd_list_local::integer_type; + using simd_type = typename simd_list_local::type::type; + static constexpr uint64_t bitsPerByte = sizeof(integer_type) * 8 / 16; + const simd_type simdValue = loadPageSafe16(&*string1Start); + const simd_type simdValues01 = simd::gatherValue('"'); + const simd_type simdValues02 = simd::gatherValue('\\'); + const simd_type simdValues03 = simd::gatherValue(static_cast(32)); + const integer_type lengthMask = static_cast((uint64_t{ 1 } << (bitsPerByte * length)) - 1); + const integer_type escapeMask = static_cast(escapeBits(simdValue, simdValues01, simdValues02, simdValues03) & lengthMask); + if (escapeMask == 0) [[likely]] { + simd::storeU(simdValue, string2); + string1Start += length; + return string2 + length; + } + return escapeLane(string1Start, string1Start + length, string2, escapeMask); + } + + template + JSONIFIER_INLINE static bool copyIfClean(basic_iterator01 string1Start, basic_iterator02 string2, const uint64_t length) noexcept { + using simd_list_local = type_list_element_t; + using integer_type = typename simd_list_local::integer_type; + using simd_type = typename simd_list_local::type::type; + static constexpr uint64_t bytesProcessed = simd_list_local::bytesProcessed; + const simd_type simdValues01 = simd::gatherValue('"'); + const simd_type simdValues02 = simd::gatherValue('\\'); + const simd_type simdValues03 = simd::gatherValue(static_cast(32)); + const simd_type head = simd::gatherValuesU(string1Start); + const simd_type tail = simd::gatherValuesU(string1Start + (length - bytesProcessed)); + const integer_type escapeMask = static_cast( + escapeBits(head, simdValues01, simdValues02, simdValues03) | escapeBits(tail, simdValues01, simdValues02, simdValues03)); + if (escapeMask != 0) [[unlikely]] { + return false; + } + simd::storeU(head, string2); + simd::storeU(tail, string2 + (length - bytesProcessed)); + return true; + } + + template + JSONIFIER_INLINE static basic_iterator02 smallImpl(basic_iterator01& string1Start, const uint64_t length, basic_iterator02 string2) noexcept { + if (length >= 4) { + uint32_t lo; + uint32_t hi; + pow2MemcpyWrapper<4>(&lo, string1Start); + pow2MemcpyWrapper<4>(&hi, string1Start + length - 4); + uint64_t word = static_cast(lo) | (static_cast(hi) << 32); + if constexpr (std::endian::native == std::endian::big) { + word = internal::byteswap(word); + } + if (!flagMask(word)) { + pow2MemcpyWrapper<4>(string2, &lo); + pow2MemcpyWrapper<4>(string2 + length - 4, &hi); + string1Start += length; + return string2 + length; + } + } else if (length >= 2) { + uint16_t lo; + uint16_t hi; + pow2MemcpyWrapper<2>(&lo, string1Start); + pow2MemcpyWrapper<2>(&hi, string1Start + length - 2); + uint64_t word = static_cast(lo) | (static_cast(hi) << 16) | 0x8080808000000000ULL; + if constexpr (std::endian::native == std::endian::big) { + word = internal::byteswap(word); + } + if (!flagMask(word)) { + pow2MemcpyWrapper<2>(string2, &lo); + pow2MemcpyWrapper<2>(string2 + length - 2, &hi); + string1Start += length; + return string2 + length; + } + } else if (length == 1) { + const uint8_t nextChar = static_cast(*string1Start); + if (escapeTable[nextChar][7] == 0) { + *string2 = nextChar; + ++string1Start; + return string2 + 1; + } + } + const basic_iterator01 string1End = string1Start + length; + return scalarImpl(string1Start, string1End, string2); + } + +#if JSONIFIER_ASAN_ENABLED + template JSONIFIER_NOINLINE JSONIFIER_NO_SANITIZE_ADDRESS static simd_type loadPageSafe16(const void* source) noexcept { + #if JSONIFIER_COMPILER_MSVC + using unaligned_uint64 = uint64_t; + #else + using unaligned_uint64 [[gnu::may_alias, gnu::aligned(1)]] = uint64_t; + #endif + const unaligned_uint64* sourceNew = static_cast(source); + const uint64_t values[2]{ sourceNew[0], sourceNew[1] }; + simd_type result; + pow2MemcpyWrapper<16>(&result, values); + return result; + } +#else + template JSONIFIER_INLINE static simd_type loadPageSafe16(const void* source) noexcept { + return simd::gatherValuesU(source); + } +#endif + + template JSONIFIER_INLINE static uint64_t escapedSize(basic_iterator stringStart, uint64_t length) noexcept { + uint64_t requiredSize{}; + const auto stringEnd = stringStart + length; + while (stringEnd - stringStart >= 8) { + uint64_t word; + pow2MemcpyWrapper<8>(&word, stringStart); + if (!flagMask(word)) { + requiredSize += 8; + } else { + for (uint64_t index{}; index < 8; ++index) { + requiredSize += escapedByteSize(static_cast(stringStart[index])); + } + } + stringStart += 8; + } + for (; stringStart < stringEnd; ++stringStart) { + requiredSize += escapedByteSize(static_cast(*stringStart)); + } + return requiredSize; + } + + template + JSONIFIER_INLINE static basic_iterator02 impl(basic_iterator01 string1Start, basic_iterator02 string2, uint64_t lengthNew) noexcept { + if (lengthNew < 16) { + if ((static_cast(std::bit_cast(&*string1Start)) & (pageSize - 1)) <= pageSize - 16) [[likely]] { + return pageSafeShortImpl(string1Start, lengthNew, string2); + } + if (lengthNew < 8) { + return smallImpl(string1Start, lengthNew, string2); + } + return swarFinish(string1Start, string1Start + lengthNew, string2); + } + if (lengthNew <= 64 && mediumImpl(string1Start, string2, lengthNew)) [[likely]] { + return string2 + lengthNew; + } + const basic_iterator01 string1End = string1Start + lengthNew; + string_parse_executor>::impl(string1Start, string1End, string2); + return overlappedFinish(string1Start, string1End, string2); + } + + template + JSONIFIER_INLINE static bool mediumImpl(basic_iterator01 string1Start, basic_iterator02 string2, const uint64_t length) noexcept { + if (length <= 32) { + return copyIfClean<2>(string1Start, string2, length); + } + if constexpr (startIndex <= 1) { + return copyIfClean<1>(string1Start, string2, length); + } else { + return copyIfClean<2>(string1Start, string2, 32) && copyIfClean<2>(string1Start + (length - 32), string2 + (length - 32), 32); + } + } + + // Sampled from Dr. Lemire's library, simdjson: https://github.com/simdjson/simdjson + template + JSONIFIER_INLINE static void copyUpTo16(basic_iterator02 destination, basic_iterator01 source, uint64_t length) noexcept { + using value_type = remove_cvref_t; + if (length >= 8) { + pow2MemcpyWrapper<8>(destination, source); + pow2MemcpyWrapper<8>(destination + length - 8, source + length - 8); + } else if (length >= 4) { + pow2MemcpyWrapper<4>(destination, source); + pow2MemcpyWrapper<4>(destination + length - 4, source + length - 4); + } else if (length > 0) { + destination[0] = static_cast(source[0]); + destination[length >> 1] = static_cast(source[length >> 1]); + destination[length - 1] = static_cast(source[length - 1]); + } + } + + JSONIFIER_INLINE static uint64_t escapedByteSize(uint8_t nextChar) noexcept { + const uint64_t escapeSize = static_cast(escapeTable[nextChar][7]); + return escapeSize > 0 ? escapeSize : 1; + } + + JSONIFIER_INLINE static uint64_t flagMask(uint64_t word) noexcept { + static constexpr uint64_t lo7Mask = 0x7F7F7F7F7F7F7F7FULL; + static constexpr uint64_t highBits = 0x8080808080808080ULL; + static constexpr uint64_t quotes = 0x2222222222222222ULL; + static constexpr uint64_t slashes = 0x5C5C5C5C5C5C5C5CULL; + static constexpr uint64_t bits56 = 0x6060606060606060ULL; + const uint64_t lo7 = word & lo7Mask; + const uint64_t quote = (lo7 ^ quotes) + lo7Mask; + const uint64_t backslash = (lo7 ^ slashes) + lo7Mask; + const uint64_t less32 = (word & bits56) + lo7Mask; + return ~((quote & backslash & less32) | word) & highBits; + } + }; + + template consteval convert_length_to_int_t getStringAsInt() noexcept { + const auto* stringNew = string.data(); + convert_length_to_int_t returnValue{}; + for (uint64_t x = 0; x < string.size(); ++x) { + returnValue |= static_cast>(stringNew[x]) << x * 8; + } + if constexpr (std::endian::native == std::endian::big) { + returnValue = internal::byteswap(returnValue); + } + return returnValue; + } + + template JSONIFIER_INLINE bool compareStringAsInt(read_buffer_ptr src) noexcept { + using integer_type = convert_length_to_int_t; + static constexpr auto string{ stringNew }; + static_assert(stringNew.size() == 4, "Sorry, but please only use a string with a length of 4 in this function!"); + static constexpr auto stringInt{ getStringAsInt() }; + integer_type sourceVal; + pow2MemcpyWrapper(&sourceVal, src); + return !static_cast(sourceVal ^ stringInt); + } + + JSONIFIER_INLINE bool validateBool(read_buffer_ptr context, read_buffer_ptr end) noexcept { + const auto remaining = end - context; + if (remaining >= 4 && compareStringAsInt<"true">(context)) { + return true; + } else if (remaining >= 5 && compareStringAsInt<"fals">(context) && context[4] == uint_c_v<'e'>) { + return true; + } + return false; + } + + JSONIFIER_INLINE bool validateNull(read_buffer_ptr context, read_buffer_ptr end) noexcept { + if (end - context >= 4 && compareStringAsInt<"null">(context)) [[likely]] { + return true; + } else { + return false; + } + } + +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/string_view.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/string_view.hpp new file mode 100644 index 000000000000..7be8710e80dc --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/string_view.hpp @@ -0,0 +1,215 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/string_view.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier { + + class string_view_base { + public: + using value_type = char; + using const_pointer = const value_type*; + using const_reference = const value_type&; + using iterator = internal::basic_iterator; + using const_iterator = internal::basic_iterator; + using difference_type = std::ptrdiff_t; + using const_reverse_iterator = std::reverse_iterator; + using size_type = uint64_t; + using traits_type = std::char_traits; + static constexpr size_type npos{ std::numeric_limits::max() }; + + JSONIFIER_INLINE static constexpr size_type fromStdPosition(size_t position) noexcept { + return position == std::basic_string_view::npos ? npos : static_cast(position); + } + + JSONIFIER_INLINE constexpr string_view_base() noexcept : dataVal{}, sizeVal{ 0 } { + } + + template> + JSONIFIER_INLINE constexpr string_view_base(const value_type_newer& stringNew) noexcept { + *this = stringNew; + } + + JSONIFIER_INLINE constexpr string_view_base(const_pointer pointerNew) noexcept : dataVal(pointerNew), sizeVal(std::char_traits::length(pointerNew)) { + } + + template JSONIFIER_INLINE constexpr string_view_base(value_type_newer&& stringNew) noexcept { + *this = stringNew; + } + + JSONIFIER_INLINE constexpr string_view_base(const_pointer pointerNew, const size_type countNew) noexcept : dataVal(pointerNew), sizeVal(countNew) { + } + + template JSONIFIER_INLINE constexpr string_view_base(const string_base& stringNew) noexcept { + *this = stringNew; + } + + template JSONIFIER_INLINE friend bool operator==(const string_view_base& lhs, const value_type_newer& rhs) noexcept { + if (lhs.size() == rhs.size()) { + if (lhs.size() > 0) { + return internal::comparison::compare(lhs.data(), rhs.data(), rhs.size()); + } + return true; + } else { + return false; + } + } + + template JSONIFIER_INLINE constexpr explicit operator std::basic_string_view() const noexcept { + return { data(), static_cast(size()) }; + } + + template JSONIFIER_INLINE constexpr string_view_base& operator=(value_type_newer&& stringNew) noexcept { + dataVal = stringNew.data(); + sizeVal = stringNew.size(); + return *this; + } + + template JSONIFIER_INLINE constexpr explicit operator std::basic_string() const { + std::basic_string returnValue{}; + returnValue.resize(sizeVal); + if (sizeVal > 0 && dataVal) [[likely]] { + internal::jsonifierMemcpy(returnValue.data(), data(), returnValue.size()); + } + return returnValue; + } + + JSONIFIER_INLINE constexpr size_type maxSize() const noexcept { + return internal::min(static_cast(std::numeric_limits::max()), static_cast(-1) / sizeof(value_type)); + } + + template JSONIFIER_INLINE friend bool operator==(const string_view_base& lhs, const char (&rhs)[size]) noexcept { + auto rhsLength = traits_type::length(rhs); + return rhsLength == lhs.size() && internal::comparison::compare(lhs.data(), rhs, rhsLength); + } + + template JSONIFIER_INLINE constexpr string_view_base& operator=(const string_base& stringNew) noexcept { + dataVal = stringNew.data(); + sizeVal = stringNew.size(); + return *this; + } + + template JSONIFIER_INLINE constexpr size_type findFirstNotOf(arg_types&&... args) const noexcept { + return fromStdPosition(this->operator std::basic_string_view().find_first_not_of(std::forward(args)...)); + } + + template JSONIFIER_INLINE constexpr size_type findLastNotOf(arg_types&&... args) const noexcept { + return fromStdPosition(this->operator std::basic_string_view().find_last_not_of(std::forward(args)...)); + } + + template JSONIFIER_INLINE constexpr size_type findFirstOf(arg_types&&... args) const noexcept { + return fromStdPosition(this->operator std::basic_string_view().find_first_of(std::forward(args)...)); + } + + template JSONIFIER_INLINE constexpr size_type findLastOf(arg_types&&... args) const noexcept { + return fromStdPosition(this->operator std::basic_string_view().find_last_of(std::forward(args)...)); + } + + JSONIFIER_INLINE constexpr string_view_base substr(const size_type offsetNew = 0, size_type countNew = npos) const { + if (offsetNew > sizeVal) [[unlikely]] { + JSONIFIER_THROW(std::out_of_range("Substring position is out of range.")); + } + + countNew = internal::min(countNew, sizeVal - offsetNew); + return string_view_base(dataVal + offsetNew, countNew); + } + + template JSONIFIER_INLINE constexpr size_type rfind(arg_types&&... args) const noexcept { + return fromStdPosition(this->operator std::basic_string_view().rfind(std::forward(args)...)); + } + + template JSONIFIER_INLINE constexpr size_type find(arg_types&&... args) const noexcept { + return fromStdPosition(this->operator std::basic_string_view().find(std::forward(args)...)); + } + + template JSONIFIER_INLINE constexpr explicit operator string_base() const noexcept { + string_base returnValue{}; + returnValue.resize(sizeVal); + if (sizeVal > 0 && dataVal) [[likely]] { + jsonifierMemcpy(returnValue.data(), data(), returnValue.size()); + } + return returnValue; + } + + template> + JSONIFIER_INLINE constexpr string_view_base& operator=(const value_type_newer& stringNew) noexcept { + dataVal = stringNew.data(); + sizeVal = stringNew.size(); + return *this; + } + + JSONIFIER_INLINE constexpr const_reference at(const size_type offsetNew) const noexcept(false) { + if (offsetNew >= sizeVal) [[unlikely]] { + JSONIFIER_THROW(std::out_of_range{ "Sorry, but that index is beyond the end of this string_view instance." }); + } + return dataVal[offsetNew]; + } + + JSONIFIER_INLINE constexpr const_reference operator[](const size_type offsetNew) const noexcept { + return dataVal[offsetNew]; + } + + JSONIFIER_INLINE constexpr const_reverse_iterator rbegin() const noexcept { + return const_reverse_iterator{ end() }; + } + + JSONIFIER_INLINE constexpr const_reverse_iterator rend() const noexcept { + return const_reverse_iterator{ begin() }; + } + + JSONIFIER_INLINE constexpr void swap(string_view_base& other) noexcept { + internal::swap(dataVal, other.dataVal); + internal::swap(sizeVal, other.sizeVal); + } + + JSONIFIER_INLINE constexpr const_iterator begin() const noexcept { + return const_iterator{ dataVal }; + } + + JSONIFIER_INLINE constexpr const_iterator end() const noexcept { + return const_iterator{ dataVal + sizeVal }; + } + + JSONIFIER_INLINE constexpr const_pointer data() const noexcept { + return dataVal; + } + + JSONIFIER_INLINE constexpr size_type length() const noexcept { + return sizeVal; + } + + JSONIFIER_INLINE constexpr size_type size() const noexcept { + return sizeVal; + } + + JSONIFIER_INLINE constexpr bool empty() const noexcept { + return sizeVal == 0; + } + + const_pointer dataVal{}; + size_type sizeVal{}; + }; + + using string_view = string_view_base; + + template JSONIFIER_INLINE internal::basic_stream& operator<<(internal::basic_stream& os, const string_view& input) noexcept { + os << std::basic_string_view{ input.data(), static_cast(input.size()) }; + return os; + } + + JSONIFIER_INLINE std::ostream& operator<<(std::ostream& os, const string_view& input) noexcept { + os << std::basic_string_view{ input.data(), static_cast(input.size()) }; + return os; + } + + JSONIFIER_INLINE constexpr string_view operator""_sv(const char* stringNew, size_t lengthNew) noexcept { + return string_view(stringNew, lengthNew); + } + +} diff --git a/deps/jsonifier/include/jsonifier-incl/utilities/utility.hpp b/deps/jsonifier/include/jsonifier-incl/utilities/utility.hpp new file mode 100644 index 000000000000..7097e4756568 --- /dev/null +++ b/deps/jsonifier/include/jsonifier-incl/utilities/utility.hpp @@ -0,0 +1,483 @@ +/* + * SPDX-License-Identifier: MIT + * Copyright (c) 2026 Nihilai Collective Corp + * https://github.com/nihilai-collective/jsonifier + * include/jsonifier-incl/utilities/utility.hpp + */ +#pragma once + +#include +#include + +namespace jsonifier::internal { + + template struct uint_c { + static constexpr uint8_t value{ static_cast(valueNew) }; + }; + + template inline constexpr uint8_t uint_c_v{ uint_c::value }; + + template using base_t = remove_cvref_t; + + consteval array genWhitespaceTable() noexcept { + array returnValues{}; + returnValues[uint_c_v<'\t'>] = true; + returnValues[uint_c_v<' '>] = true; + returnValues[uint_c_v<'\n'>] = true; + returnValues[uint_c_v<'\r'>] = true; + return returnValues; + } + + alignas(64) inline constexpr auto whitespaceTableRaw{ genWhitespaceTable() }; + + inline constexpr const bool* __restrict whitespaceTable{ whitespaceTableRaw.data() }; + + consteval array genNestingDeltaTable() noexcept { + array returnValues{}; + returnValues[uint_c_v<'{'>] = 1; + returnValues[uint_c_v<'['>] = 1; + returnValues[uint_c_v<'}'>] = -1; + returnValues[uint_c_v<']'>] = -1; + return returnValues; + } + + alignas(64) inline constexpr auto nestingDeltaTableRaw{ genNestingDeltaTable() }; + + inline constexpr const int8_t* __restrict nestingDeltaTable{ nestingDeltaTableRaw.data() }; + + consteval array genNewlineTable() noexcept { + array returnValues{}; + returnValues[uint_c_v<'\n'>] = true; + returnValues[uint_c_v<'\r'>] = true; + return returnValues; + } + + alignas(64) inline constexpr auto newlineTableRaw{ genNewlineTable() }; + + inline constexpr const bool* __restrict newlineTable{ newlineTableRaw.data() }; + + consteval array genNumberTable() noexcept { + array returnValues{}; + returnValues[uint_c_v<'-'>] = true; + returnValues[uint_c_v<'0'>] = true; + returnValues[uint_c_v<'1'>] = true; + returnValues[uint_c_v<'2'>] = true; + returnValues[uint_c_v<'3'>] = true; + returnValues[uint_c_v<'4'>] = true; + returnValues[uint_c_v<'5'>] = true; + returnValues[uint_c_v<'6'>] = true; + returnValues[uint_c_v<'7'>] = true; + returnValues[uint_c_v<'8'>] = true; + returnValues[uint_c_v<'9'>] = true; + return returnValues; + } + + alignas(64) inline constexpr auto numberTableRaw{ genNumberTable() }; + + inline constexpr const bool* __restrict numberTable{ numberTableRaw.data() }; + + struct alignas(8) char_escape_entry { + uint64_t size{}; + char values[8]{}; + }; + + consteval array genCharEscapeEntries() noexcept { + constexpr char hexDigits[]{ "0123456789ABCDEF" }; + array returnValue{}; + for (uint64_t x = 0; x < 256ULL; ++x) { + if (x < 0x20) { + returnValue[x].values[0] = '\\'; + returnValue[x].values[1] = 'u'; + returnValue[x].values[2] = '0'; + returnValue[x].values[3] = '0'; + returnValue[x].values[4] = hexDigits[(x >> 4) & 0xF]; + returnValue[x].values[5] = hexDigits[x & 0xF]; + returnValue[x].size = 6; + } else if (x < 0x80) { + returnValue[x].values[0] = static_cast(x); + returnValue[x].size = 1; + } else { + returnValue[x].values[0] = static_cast(0xEF); + returnValue[x].values[1] = static_cast(0xBF); + returnValue[x].values[2] = static_cast(0xBD); + returnValue[x].size = 3; + } + } + auto setEscape = [&](char value, char escaped) { + auto& entry = returnValue[static_cast(value)]; + entry = char_escape_entry{}; + entry.values[0] = '\\'; + entry.values[1] = escaped; + entry.size = 2; + }; + setEscape('\b', 'b'); + setEscape('\t', 't'); + setEscape('\n', 'n'); + setEscape('\f', 'f'); + setEscape('\r', 'r'); + setEscape('\"', '\"'); + setEscape('\\', '\\'); + return returnValue; + } + + alignas(64) inline constexpr auto charEscapeEntriesRaw{ genCharEscapeEntries() }; + + inline constexpr const char_escape_entry* charEscapeEntries{ charEscapeEntriesRaw.data() }; + + template JSONIFIER_INLINE constexpr value_type&& forward(remove_reference_t& t JSONIFIER_LIFETIME_BOUND) noexcept { + return static_cast(t); + } + + template JSONIFIER_INLINE constexpr remove_reference_t&& move(value_type&& value) noexcept { + return static_cast&&>(value); + } + + template + requires(std::is_rvalue_reference_v) + JSONIFIER_INLINE constexpr value_type&& forward(remove_reference_t&& t) noexcept { + static_assert(!std::is_lvalue_reference_v, "value_type cannot be an lvalue reference (e.g., U&)."); + return static_cast(t); + } + + struct swap_fn { + template JSONIFIER_INLINE constexpr void operator()(value_type (&lhs)[size], value_type (&rhs)[size]) const + noexcept(noexcept((*this)(lhs[0], rhs[0]))) { + for (std::size_t index = 0; index < size; ++index) { + (*this)(lhs[index], rhs[index]); + } + } + + template + requires(std::is_move_constructible_v && std::is_move_assignable_v) + JSONIFIER_INLINE constexpr void operator()(value_type& lhs, value_type& rhs) const + noexcept(std::is_nothrow_move_constructible_v && std::is_nothrow_move_assignable_v) { + value_type temp{ internal::move(lhs) }; + lhs = internal::move(rhs); + rhs = internal::move(temp); + } + }; + + inline constexpr swap_fn swap{}; + + template struct type_holder { + static constexpr uint64_t bytesProcessed{ bytesProcessedNew }; + static constexpr integer_type_new mask{ maskNew }; + using type = simd_type; + using integer_type = integer_type_new; + }; + + template struct get_int_type { + using type = conditional_t, uint8_t, int8_t>; + }; + + template struct integral_constant { + static constexpr uint64_t value = valueNew; + + JSONIFIER_INLINE constexpr uint64_t operator()() const noexcept { + return value; + } + + JSONIFIER_INLINE constexpr operator uint64_t() const noexcept { + return value; + } + }; + + template using tag = integral_constant; + + template struct integer_sequence {}; + + template struct concat_sequence; + + template struct concat_sequence, integer_sequence> { + using type = integer_sequence; + }; + + template struct make_sequence_impl { + using half_type = typename make_sequence_impl::type; + using remainder_type = typename make_sequence_impl::type; + using type = typename concat_sequence::type; + }; + + template<> struct make_sequence_impl<0ULL> { + using type = integer_sequence<>; + }; + + template<> struct make_sequence_impl<1ULL> { + using type = integer_sequence<0>; + }; + + template using make_integer_sequence = typename make_sequence_impl::type; + + template struct offset_sequence; + + template struct offset_sequence, offset> { + using type = integer_sequence; + }; + + template struct step_sequence; + + template struct step_sequence, stepNew> { + using type = integer_sequence; + }; + + template using step_sequence_t = typename step_sequence::type; + + template + requires(end >= start && step > 0) + using make_stepped_range_sequence = typename offset_sequence, step>, start>::type; + + template typename functor_type, typename integer_sequence, auto...> struct functor_runner; + + template typename functor_type, uint64_t... indices, auto... values> struct functor_runner, values...> { + template JSONIFIER_INLINE static auto implAnd([[maybe_unused]] arg_types&&... args) noexcept { + return (functor_type::template impl(internal::forward(args)...) && ...); + } + + template JSONIFIER_INLINE static auto impl([[maybe_unused]] arg_types&&... args) noexcept { + return (functor_type::template impl(internal::forward(args)...), ...); + } + }; + + template typename functor_type, uint64_t... indices, uint64_t offsetVal, auto... values> + struct functor_runner, offsetVal>, values...> { + template JSONIFIER_INLINE static auto implAnd([[maybe_unused]] arg_types&&... args) noexcept { + return (functor_type::template impl(internal::forward(args)...) && ...); + } + + template JSONIFIER_INLINE static auto impl([[maybe_unused]] arg_types&&... args) noexcept { + return (functor_type::template impl(internal::forward(args)...), ...); + } + }; + + template struct visit_impl; + + template struct visit_impl> { + template JSONIFIER_INLINE static constexpr void impl(variant_type&& variant, arg_types&&... args) noexcept { + const auto idx = variant.index(); + static_cast(( + (idx == indices ? (function_type::impl(std::get(internal::forward(variant)), internal::forward(args)...), true) : false) || ...)); + } + }; + + template JSONIFIER_INLINE constexpr void visit(variant_type&& variant, arg_types&&... args) noexcept { + visit_impl>>>::impl(internal::forward(variant), + internal::forward(args)...); + } + + template JSONIFIER_INLINE constexpr value_type01 max(value_type01 value1, value_type02 value2) noexcept { + return value1 > static_cast(value2) ? value1 : static_cast(value2); + } + + template JSONIFIER_INLINE constexpr value_type01 min(value_type01 value1, value_type02 value2) noexcept { + return value1 < static_cast(value2) ? value1 : static_cast(value2); + } + + JSONIFIER_INLINE constexpr uint64_t strLen(const char* input) noexcept { + uint64_t returnVal{}; + if (input) { + while (input[returnVal] != '\0') { + ++returnVal; + } + } + return returnVal; + } + + template struct digit_sizes; + + template struct digit_sizes { + static constexpr uint64_t value{ 20 }; + }; + + template struct digit_sizes { + static constexpr uint64_t value{ 20 }; + }; + + template struct digit_sizes { + static constexpr uint64_t value{ 10 }; + }; + + template struct digit_sizes { + static constexpr uint64_t value{ 11 }; + }; + + template struct digit_sizes { + static constexpr uint64_t value{ 5 }; + }; + + template struct digit_sizes { + static constexpr uint64_t value{ 6 }; + }; + + template struct digit_sizes { + static constexpr uint64_t value{ 3 }; + }; + + template struct digit_sizes { + static constexpr uint64_t value{ 4 }; + }; + + template struct digit_sizes { + static constexpr uint64_t value{ 17 }; + }; + + template struct digit_sizes { + static constexpr uint64_t value{ 34 }; + }; + + template inline constexpr uint64_t digit_size_v = digit_sizes::value; + + template struct char_sizes { + static constexpr uint64_t value{ 16 }; + }; + + template inline constexpr uint64_t char_size_v = char_sizes::value; + +#if JSONIFIER_PLATFORM_MAC + + template JSONIFIER_INLINE void copyOverlappingChunks(std::byte* __restrict dst, const std::byte* __restrict src, uint64_t byteCount) { + pow2MemcpyWrapper(dst, src); + pow2MemcpyWrapper(dst + byteCount - chunkBytes, src + byteCount - chunkBytes); + } + + template JSONIFIER_INLINE void jsonifierMemcpyUpTo(void* __restrict dst, const void* __restrict src, uint64_t byteCount) { + switch (std::bit_width(byteCount)) { + case 0: { + return; + } + case 1: { + *static_cast(dst) = *static_cast(src); + return; + } + case 2: { + if constexpr (maxBytes >= 2) { + copyOverlappingChunks<2>(static_cast(dst), static_cast(src), byteCount); + } + return; + } + case 3: { + if constexpr (maxBytes >= 4) { + copyOverlappingChunks<4>(static_cast(dst), static_cast(src), byteCount); + } + return; + } + case 4: { + if constexpr (maxBytes >= 8) { + copyOverlappingChunks<8>(static_cast(dst), static_cast(src), byteCount); + } + return; + } + case 5: { + if constexpr (maxBytes >= 16) { + copyOverlappingChunks<16>(static_cast(dst), static_cast(src), byteCount); + } + return; + } + case 6: { + if constexpr (maxBytes >= 32) { + copyOverlappingChunks<32>(static_cast(dst), static_cast(src), byteCount); + } + return; + } + default: { + if constexpr (maxBytes >= 64) { + memcpyWrapper(static_cast(dst), static_cast(src), byteCount); + } + return; + } + } + } + + JSONIFIER_INLINE void jsonifierMemcpy(void* __restrict destination, const void* __restrict source, uint64_t byteCount) { + jsonifierMemcpyUpTo::max()>(destination, source, byteCount); + } + +#else + + template struct memcpy_runner_impl; + + inline constexpr uint64_t registerWidth{ cpu_properties::getValue(cpu_property_types::alignment) }; + + template struct memcpy_runner_impl> { + static_assert(std::has_single_bit(bucketSize), "Sorry, but buckets must be a power-of-2 size."); + static constexpr uint64_t chunkSize{ bucketSize < registerWidth ? bucketSize : registerWidth }; + static constexpr uint64_t chunkCount{ bucketSize / chunkSize }; + + JSONIFIER_INLINE static void impl(std::byte* __restrict dst, const std::byte* __restrict src, uint64_t byteCount) noexcept { + std::byte* __restrict dstTail = dst + byteCount - bucketSize; + const std::byte* __restrict srcTail = src + byteCount - bucketSize; + (pow2MemcpyWrapper(dst + indices * chunkSize, src + indices * chunkSize), ...); + (pow2MemcpyWrapper(dstTail + indices * chunkSize, srcTail + indices * chunkSize), ...); + } + }; + + template struct memcpy_runner { + static_assert(std::has_single_bit(bucketSize), "Sorry, but buckets must be a power-of-2 size."); + static constexpr uint64_t chunkSize{ bucketSize < registerWidth ? bucketSize : registerWidth }; + static constexpr uint64_t chunkCount{ bucketSize / chunkSize }; + + JSONIFIER_INLINE static void impl(std::byte* __restrict dst, const std::byte* __restrict src, uint64_t byteCount) noexcept { + memcpy_runner_impl>::impl(dst, src, byteCount); + } + }; + + inline constexpr uint64_t fallbackBucketIndex{ 9 }; + + template inline constexpr uint64_t maxBytesBucketIndex{ static_cast(std::bit_width(maxBytes)) }; + + template + inline constexpr uint64_t topBucketIndex{ maxBytesBucketIndex < fallbackBucketIndex ? maxBytesBucketIndex : fallbackBucketIndex }; + + template JSONIFIER_INLINE void copyBucket(std::byte* __restrict dst, const std::byte* __restrict src, uint64_t byteCount) noexcept { + if constexpr (bucketIndex == 1) { + *dst = *src; + } else if constexpr (bucketIndex == fallbackBucketIndex) { + memcpyWrapper(dst, src, byteCount); + } else if constexpr (bucketIndex > 1) { + memcpy_runner<1ull << (bucketIndex - 1)>::impl(dst, src, byteCount); + } + } + + template JSONIFIER_INLINE void dispatchBuckets(std::byte* __restrict dst, const std::byte* __restrict src, uint64_t byteCount) noexcept { + if constexpr (lowIndex == highIndex) { + copyBucket(dst, src, byteCount); + } else { + static constexpr uint64_t splitIndex{ (lowIndex + highIndex + 1) / 2 }; + if (byteCount < (1ull << (splitIndex - 1))) { + dispatchBuckets(dst, src, byteCount); + } else { + dispatchBuckets(dst, src, byteCount); + } + } + } + + template JSONIFIER_INLINE void jsonifierMemcpyUpTo(void* __restrict dst, const void* __restrict src, uint64_t byteCount) noexcept { + dispatchBuckets<0, topBucketIndex>(static_cast(dst), static_cast(src), byteCount); + } + + JSONIFIER_INLINE void jsonifierMemcpy(void* __restrict dst, const void* __restrict src, uint64_t byteCount) noexcept { + jsonifierMemcpyUpTo::max()>(dst, src, byteCount); + } + +#endif + +} + +#include + +namespace jsonifier::internal::simd { + +#if JSONIFIER_NEON || JSONIFIER_SVE2 + using avx_integer_list = type_list_t, uint64_t, 64>, type_holder<32, simd_type_wrapper, uint32_t, 32>, + type_holder<16, simd_type_wrapper, uint64_t, 16>>; + using avx_list = type_list_t, uint64_t, std::numeric_limits::max()>, + type_holder<32, simd_type_wrapper, uint32_t, std::numeric_limits::max()>, + type_holder<16, simd_type_wrapper, uint64_t, std::numeric_limits::max()>>; +#else + using avx_integer_list = type_list_t, uint64_t, 64>, type_holder<32, simd_type_wrapper, uint32_t, 32>, + type_holder<16, simd_type_wrapper, uint16_t, 16>>; + using avx_list = type_list_t, uint64_t, std::numeric_limits::max()>, + type_holder<32, simd_type_wrapper, uint32_t, std::numeric_limits::max()>, + type_holder<16, simd_type_wrapper, uint16_t, std::numeric_limits::max()>>; +#endif + +} diff --git a/deps/jsonifier/json_walker-inl.h b/deps/jsonifier/json_walker-inl.h new file mode 100644 index 000000000000..39cebc49d212 --- /dev/null +++ b/deps/jsonifier/json_walker-inl.h @@ -0,0 +1,149 @@ +#ifndef DEPS_JSONIFIER_JSON_WALKER_INL_H_ +#define DEPS_JSONIFIER_JSON_WALKER_INL_H_ + +namespace node { +namespace modules { +namespace NODE_JSONIFIER_ISA_NS { + +using GenericDocument = jsonifier::generic::generic_document; +using GenericObject = jsonifier::generic::generic_object; +using GenericArray = jsonifier::generic::generic_array; +using GenericField = jsonifier::generic::generic_field; +using GenericValue = jsonifier::generic::generic_value; + +class JsonWalker { + public: + static std::string_view Walk(const GenericValue& value, + json::JsonEventSink* sink, + size_t depth) { + if (depth > kMaxDepth) { + return kTooDeep; + } + switch (value.type()) { + case jsonifier::json_types::object: + return WalkObject(value, sink, depth); + case jsonifier::json_types::array: + return WalkArray(value, sink, depth); + case jsonifier::json_types::string: { + std::string_view text; + if (const auto code = value.getString(text); code != kSuccess) { + return Name(code); + } + sink->String(text); + return {}; + } + case jsonifier::json_types::number: { + jsonifier::string_view raw{}; + double parsed; + if (const auto code = value.rawJson(raw); code != kSuccess) { + return Name(code); + } + if (const auto code = value.getDouble(parsed); code != kSuccess) { + return Name(code); + } + sink->Number(ToView(raw), parsed); + return {}; + } + case jsonifier::json_types::boolean: { + bool parsed; + if (const auto code = value.getBool(parsed); code != kSuccess) { + return Name(code); + } + sink->Bool(parsed); + return {}; + } + case jsonifier::json_types::null: + if (!value.isNull()) { + return Name(jsonifier::generic::error_codes::n_atom_error); + } + sink->Null(); + return {}; + default: + return Name(jsonifier::generic::error_codes::tape_error); + } + } + + static std::string_view Name(jsonifier::generic::error_codes code) { + return jsonifier::internal::getName(code) + .operator std::basic_string_view(); + } + + private: + static constexpr size_t kMaxDepth = 1024; + static constexpr std::string_view kTooDeep = "depth_limit"; + static constexpr jsonifier::generic::error_codes kSuccess = + jsonifier::generic::error_codes::success; + + static std::string_view WalkObject(const GenericValue& value, + json::JsonEventSink* sink, + size_t depth) { + GenericObject object = value.getObject(); + if (object.error() != kSuccess) { + return Name(object.error()); + } + sink->StartObject(); + for (GenericField field : object) { + std::string_view key; + if (field.error() != kSuccess) { + return Name(field.error()); + } + if (const auto code = field.unescapedKey(key); code != kSuccess) { + return Name(code); + } + sink->Key(key); + if (const std::string_view error = Walk(field.value(), sink, depth + 1); + !error.empty()) { + return error; + } + } + jsonifier::string_view raw{}; + if (const auto code = value.rawJson(raw); code != kSuccess) { + return Name(code); + } + sink->EndObject(ToView(raw)); + return {}; + } + + static std::string_view WalkArray(const GenericValue& value, + json::JsonEventSink* sink, + size_t depth) { + GenericArray array = value.getArray(); + if (array.error() != kSuccess) { + return Name(array.error()); + } + sink->StartArray(); + for (GenericValue element : array) { + if (element.error() != kSuccess) { + return Name(element.error()); + } + if (const std::string_view error = Walk(element, sink, depth + 1); + !error.empty()) { + return error; + } + } + jsonifier::string_view raw{}; + if (const auto code = value.rawJson(raw); code != kSuccess) { + return Name(code); + } + sink->EndArray(ToView(raw)); + return {}; + } +}; + +std::string_view WalkJson(std::string_view json, json::JsonEventSink* sink) { + if (json.starts_with(kJsonifierUtf8Bom)) { + json.remove_prefix(kJsonifierUtf8Bom.size()); + } + jsonifier::jsonifier_core parser; + GenericDocument document = parser.iterate(json); + if (document.error() != jsonifier::generic::error_codes::success) { + return JsonWalker::Name(document.error()); + } + return JsonWalker::Walk(document, sink, 0); +} + +} // namespace NODE_JSONIFIER_ISA_NS +} // namespace modules +} // namespace node + +#endif // DEPS_JSONIFIER_JSON_WALKER_INL_H_ diff --git a/deps/jsonifier/jsonifier.gyp b/deps/jsonifier/jsonifier.gyp new file mode 100644 index 000000000000..35e9817a0ff2 --- /dev/null +++ b/deps/jsonifier/jsonifier.gyp @@ -0,0 +1,70 @@ +{ + 'variables': { + 'jsonifier_include_dir%': 'include', + }, + 'targets': [ + { + 'target_name': 'jsonifier', + 'includes': [ 'jsonifier_target.gypi' ], + 'sources': [ + 'package_json_parser.cc', + ], + 'conditions': [ + ['target_arch=="x64"', { + 'dependencies': [ 'jsonifier_avx2', 'jsonifier_avx512' ], + 'sources': [ + 'package_json_parser_fallback.cc', + ], + }], + ['target_arch=="arm64"', { + 'sources': [ + 'package_json_parser_neon.cc', + ], + }], + ['target_arch!="x64" and target_arch!="arm64"', { + 'sources': [ + 'package_json_parser_fallback.cc', + ], + }], + ], + }, + ], + 'conditions': [ + ['target_arch=="x64"', { + 'targets': [ + { + 'target_name': 'jsonifier_avx2', + 'includes': [ 'jsonifier_target.gypi' ], + 'sources': [ + 'package_json_parser_avx2.cc', + ], + 'cflags_cc': [ '-mavx2', '-mbmi', '-mbmi2', '-mlzcnt', '-mpopcnt', '-mpclmul' ], + 'xcode_settings': { + 'OTHER_CPLUSPLUSFLAGS': [ '-mavx2', '-mbmi', '-mbmi2', '-mlzcnt', '-mpopcnt', '-mpclmul' ], + }, + 'msvs_settings': { + 'VCCLCompilerTool': { + 'EnableEnhancedInstructionSet': '5', + }, + }, + }, + { + 'target_name': 'jsonifier_avx512', + 'includes': [ 'jsonifier_target.gypi' ], + 'sources': [ + 'package_json_parser_avx512.cc', + ], + 'cflags_cc': [ '-mavx2', '-mbmi', '-mbmi2', '-mlzcnt', '-mpopcnt', '-mpclmul', '-mavx512f', '-mavx512dq', '-mavx512bw', '-mavx512vl', '-mavx512vbmi2' ], + 'xcode_settings': { + 'OTHER_CPLUSPLUSFLAGS': [ '-mavx2', '-mbmi', '-mbmi2', '-mlzcnt', '-mpopcnt', '-mpclmul', '-mavx512f', '-mavx512dq', '-mavx512bw', '-mavx512vl', '-mavx512vbmi2' ], + }, + 'msvs_settings': { + 'VCCLCompilerTool': { + 'AdditionalOptions': [ '-mavx2', '-mbmi', '-mbmi2', '-mlzcnt', '-mpopcnt', '-mpclmul', '-mavx512f', '-mavx512dq', '-mavx512bw', '-mavx512vl', '-mavx512vbmi2' ], + }, + }, + }, + ], + }], + ], +} diff --git a/deps/jsonifier/jsonifier_isa_begin.h b/deps/jsonifier/jsonifier_isa_begin.h new file mode 100644 index 000000000000..7251d511bec5 --- /dev/null +++ b/deps/jsonifier/jsonifier_isa_begin.h @@ -0,0 +1,100 @@ +#ifndef DEPS_JSONIFIER_JSONIFIER_ISA_BEGIN_H_ +#define DEPS_JSONIFIER_JSONIFIER_ISA_BEGIN_H_ + +#include "package_json_backend.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__x86_64__) || defined(_M_X64) +#include +#endif +#if defined(_MSC_VER) +#include +#endif + +#if NODE_JSONIFIER_ISA_AVX512 +#define JSONIFIER_LZCNT 1 +#define JSONIFIER_POPCNT 1 +#define JSONIFIER_BMI 1 +#define JSONIFIER_CLMUL 1 +#define JSONIFIER_AVX 1 +#define JSONIFIER_AVX2 1 +#define JSONIFIER_AVX512 1 +#define JSONIFIER_NEON 0 +#define JSONIFIER_SVE2 0 +#define JSONIFIER_CPU_ALIGNMENT 64 +#define jsonifier jsonifier_avx512 +#define NODE_JSONIFIER_ISA_NS isa_avx512 +#elif NODE_JSONIFIER_ISA_AVX2 +#define JSONIFIER_LZCNT 1 +#define JSONIFIER_POPCNT 1 +#define JSONIFIER_BMI 1 +#define JSONIFIER_CLMUL 1 +#define JSONIFIER_AVX 1 +#define JSONIFIER_AVX2 1 +#define JSONIFIER_AVX512 0 +#define JSONIFIER_NEON 0 +#define JSONIFIER_SVE2 0 +#define JSONIFIER_CPU_ALIGNMENT 32 +#define jsonifier jsonifier_avx2 +#define NODE_JSONIFIER_ISA_NS isa_avx2 +#elif NODE_JSONIFIER_ISA_NEON +#define JSONIFIER_LZCNT 0 +#define JSONIFIER_POPCNT 0 +#define JSONIFIER_BMI 0 +#define JSONIFIER_CLMUL 0 +#define JSONIFIER_AVX 0 +#define JSONIFIER_AVX2 0 +#define JSONIFIER_AVX512 0 +#define JSONIFIER_NEON 1 +#define JSONIFIER_SVE2 0 +#define JSONIFIER_CPU_ALIGNMENT 16 +#define jsonifier jsonifier_neon +#define NODE_JSONIFIER_ISA_NS isa_neon +#else +#define JSONIFIER_LZCNT 0 +#define JSONIFIER_POPCNT 0 +#define JSONIFIER_BMI 0 +#define JSONIFIER_CLMUL 0 +#define JSONIFIER_AVX 0 +#define JSONIFIER_AVX2 0 +#define JSONIFIER_AVX512 0 +#define JSONIFIER_NEON 0 +#define JSONIFIER_SVE2 0 +#define JSONIFIER_CPU_ALIGNMENT 16 +#define jsonifier jsonifier_fallback +#define NODE_JSONIFIER_ISA_NS isa_fallback +#endif + +#include + +#endif // DEPS_JSONIFIER_JSONIFIER_ISA_BEGIN_H_ diff --git a/deps/jsonifier/jsonifier_isa_end.h b/deps/jsonifier/jsonifier_isa_end.h new file mode 100644 index 000000000000..da26923a178d --- /dev/null +++ b/deps/jsonifier/jsonifier_isa_end.h @@ -0,0 +1,7 @@ +#ifndef DEPS_JSONIFIER_JSONIFIER_ISA_END_H_ +#define DEPS_JSONIFIER_JSONIFIER_ISA_END_H_ + +#undef jsonifier +#undef NODE_JSONIFIER_ISA_NS + +#endif // DEPS_JSONIFIER_JSONIFIER_ISA_END_H_ diff --git a/deps/jsonifier/jsonifier_target.gypi b/deps/jsonifier/jsonifier_target.gypi new file mode 100644 index 000000000000..28f51e7c5dbf --- /dev/null +++ b/deps/jsonifier/jsonifier_target.gypi @@ -0,0 +1,12 @@ +{ + 'type': 'static_library', + 'include_dirs': [ + '<(jsonifier_include_dir)', + '../../src', + ], + 'configurations': { + 'Release': { + 'defines': [ 'NDEBUG' ], + }, + }, +} diff --git a/deps/jsonifier/package_json_backend.h b/deps/jsonifier/package_json_backend.h new file mode 100644 index 000000000000..d9b91d453a30 --- /dev/null +++ b/deps/jsonifier/package_json_backend.h @@ -0,0 +1,60 @@ +#ifndef DEPS_JSONIFIER_PACKAGE_JSON_BACKEND_H_ +#define DEPS_JSONIFIER_PACKAGE_JSON_BACKEND_H_ + +#include + +#include "node_json_events.h" + +namespace node { +namespace modules { + +struct PackageJSONFields; + +inline constexpr std::string_view kJsonifierUtf8Bom = "\xEF\xBB\xBF"; + +class PackageJSONFieldSink { + public: + explicit PackageJSONFieldSink(PackageJSONFields* fields) : fields_(fields) {} + + void SetName(std::string_view value); + void SetMain(std::string_view value); + void SetType(std::string_view value); + void SetExports(std::string_view value); + void SetImports(std::string_view value); + void SetScripts(std::string_view value); + void SetPrivate(bool value); + + private: + PackageJSONFields* fields_; +}; + +class PackageJSONBackendParser { + public: + virtual ~PackageJSONBackendParser(); + virtual bool Parse(std::string_view json, PackageJSONFieldSink* sink) = 0; +}; + +namespace isa_avx512 { +PackageJSONBackendParser* NewGenericParser(); +std::string_view WalkJson(std::string_view json, json::JsonEventSink* sink); +} // namespace isa_avx512 + +namespace isa_avx2 { +PackageJSONBackendParser* NewGenericParser(); +std::string_view WalkJson(std::string_view json, json::JsonEventSink* sink); +} // namespace isa_avx2 + +namespace isa_fallback { +PackageJSONBackendParser* NewGenericParser(); +std::string_view WalkJson(std::string_view json, json::JsonEventSink* sink); +} // namespace isa_fallback + +namespace isa_neon { +PackageJSONBackendParser* NewGenericParser(); +std::string_view WalkJson(std::string_view json, json::JsonEventSink* sink); +} // namespace isa_neon + +} // namespace modules +} // namespace node + +#endif // DEPS_JSONIFIER_PACKAGE_JSON_BACKEND_H_ diff --git a/deps/jsonifier/package_json_generic_parser-inl.h b/deps/jsonifier/package_json_generic_parser-inl.h new file mode 100644 index 000000000000..8d977fb85714 --- /dev/null +++ b/deps/jsonifier/package_json_generic_parser-inl.h @@ -0,0 +1,191 @@ +#ifndef DEPS_JSONIFIER_PACKAGE_JSON_GENERIC_PARSER_INL_H_ +#define DEPS_JSONIFIER_PACKAGE_JSON_GENERIC_PARSER_INL_H_ + +namespace node { +namespace modules { +namespace NODE_JSONIFIER_ISA_NS { + +using GenericDocument = jsonifier::generic::generic_document; +using GenericObject = jsonifier::generic::generic_object; +using GenericArray = jsonifier::generic::generic_array; +using GenericField = jsonifier::generic::generic_field; +using GenericValue = jsonifier::generic::generic_value; + +using SinkSetter = void (PackageJSONFieldSink::*)(std::string_view); + +static std::string_view ToView(const jsonifier::string_view& value) { + return {value.data(), static_cast(value.size())}; +} + +static std::string_view ToView(const jsonifier::string& value) { + return {value.data(), static_cast(value.size())}; +} + +enum class PackageField { + kOther, + kName, + kMain, + kType, + kExports, + kImports, + kPrivate, + kScripts, +}; + +static PackageField ClassifyKey(std::string_view key) { + if (key.size() == 4) { + switch (key[0]) { + case 'n': + return key == "name" ? PackageField::kName : PackageField::kOther; + case 'm': + return key == "main" ? PackageField::kMain : PackageField::kOther; + case 't': + return key == "type" ? PackageField::kType : PackageField::kOther; + default: + return PackageField::kOther; + } + } + if (key.size() == 7) { + switch (key[0]) { + case 'e': + return key == "exports" ? PackageField::kExports : PackageField::kOther; + case 'i': + return key == "imports" ? PackageField::kImports : PackageField::kOther; + case 'p': + return key == "private" ? PackageField::kPrivate : PackageField::kOther; + case 's': + return key == "scripts" ? PackageField::kScripts : PackageField::kOther; + default: + return PackageField::kOther; + } + } + return PackageField::kOther; +} + +class GenericParser final : public PackageJSONBackendParser { + public: + bool Parse(std::string_view json, PackageJSONFieldSink* sink) override { + if (json.starts_with(kJsonifierUtf8Bom)) { + json.remove_prefix(kJsonifierUtf8Bom.size()); + } + + GenericDocument document = parser_.iterate(json); + if (document.error() != jsonifier::generic::error_codes::success) { + return false; + } + GenericObject main_object = document.getObject(); + if (main_object.error() != jsonifier::generic::error_codes::success) { + return false; + } + + for (GenericField field : main_object) { + if (field.error() != jsonifier::generic::error_codes::success) { + return false; + } + const PackageField kind = ClassifyKey(ToView(field.rawKey())); + if (kind == PackageField::kOther) { + continue; + } + GenericValue value = field.value(); + + switch (kind) { + case PackageField::kName: + if (value.getString(scratch_) != + jsonifier::generic::error_codes::success) { + return false; + } + sink->SetName(ToView(scratch_)); + break; + case PackageField::kMain: + if (value.getString(scratch_) == + jsonifier::generic::error_codes::success) { + sink->SetMain(ToView(scratch_)); + } + break; + case PackageField::kExports: + if (!ForwardContainerOrString( + value, sink, &PackageJSONFieldSink::SetExports)) { + return false; + } + break; + case PackageField::kImports: + if (!ForwardContainerOrString( + value, sink, &PackageJSONFieldSink::SetImports)) { + return false; + } + break; + case PackageField::kType: { + std::string_view type; + if (value.getString(type) != + jsonifier::generic::error_codes::success) { + return false; + } + if (type == "commonjs" || type == "module") { + sink->SetType(type); + } + break; + } + case PackageField::kPrivate: { + bool is_private; + if (value.getBool(is_private) == + jsonifier::generic::error_codes::success) { + sink->SetPrivate(is_private); + } + break; + } + case PackageField::kScripts: + if (value.type() == jsonifier::json_types::object) { + jsonifier::string_view raw{}; + if (value.rawJson(raw) != + jsonifier::generic::error_codes::success) { + return false; + } + sink->SetScripts(ToView(raw)); + } + break; + case PackageField::kOther: + break; + } + } + return true; + } + + private: + bool ForwardContainerOrString(const GenericValue& value, + PackageJSONFieldSink* sink, + SinkSetter setter) { + switch (value.type()) { + case jsonifier::json_types::object: + case jsonifier::json_types::array: { + jsonifier::string_view raw{}; + if (value.rawJson(raw) != jsonifier::generic::error_codes::success) { + return false; + } + (sink->*setter)(ToView(raw)); + return true; + } + case jsonifier::json_types::string: + if (value.getString(scratch_) != + jsonifier::generic::error_codes::success) { + return false; + } + (sink->*setter)(ToView(scratch_)); + return true; + default: + return true; + } + } + + jsonifier::jsonifier_core parser_; + jsonifier::string scratch_; +}; + +PackageJSONBackendParser* NewGenericParser() { + return new GenericParser(); +} + +} // namespace NODE_JSONIFIER_ISA_NS +} // namespace modules +} // namespace node + +#endif // DEPS_JSONIFIER_PACKAGE_JSON_GENERIC_PARSER_INL_H_ diff --git a/deps/jsonifier/package_json_parser.cc b/deps/jsonifier/package_json_parser.cc new file mode 100644 index 000000000000..eaafb5b90a42 --- /dev/null +++ b/deps/jsonifier/package_json_parser.cc @@ -0,0 +1,257 @@ +#include "node_package_json_parser.h" +#include "package_json_backend.h" + +#include +#include +#include +#include +#include + +#if defined(__x86_64__) || defined(_M_X64) +#define NODE_JSONIFIER_X64 1 +#if defined(_MSC_VER) +#include +#else +#include +#endif +#else +#define NODE_JSONIFIER_X64 0 +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#define NODE_JSONIFIER_ARM64 1 +#else +#define NODE_JSONIFIER_ARM64 0 +#endif + +namespace node { +namespace modules { + +namespace { + +bool EndsWithObjectClose(std::string_view json) { + const size_t last = json.find_last_not_of(" \t\n\r"); + return last != std::string_view::npos && json[last] == '}'; +} + +enum class JsonifierIsa { kFallback, kAvx2, kAvx512, kNeon }; + +#if NODE_JSONIFIER_X64 + +struct CpuidRegisters { + uint32_t eax; + uint32_t ebx; + uint32_t ecx; + uint32_t edx; +}; + +CpuidRegisters QueryCpuid(uint32_t leaf, uint32_t subleaf) { +#if defined(_MSC_VER) + int registers[4]; + __cpuidex(registers, static_cast(leaf), static_cast(subleaf)); + return {static_cast(registers[0]), + static_cast(registers[1]), + static_cast(registers[2]), + static_cast(registers[3])}; +#else + CpuidRegisters registers{}; + __cpuid_count(leaf, + subleaf, + registers.eax, + registers.ebx, + registers.ecx, + registers.edx); + return registers; +#endif +} + +uint64_t QueryXcr0() { +#if defined(_MSC_VER) && !defined(__clang__) + return _xgetbv(0); +#else + uint32_t low; + uint32_t high; + __asm__ volatile("xgetbv" : "=a"(low), "=d"(high) : "c"(0)); + return (static_cast(high) << 32) | low; +#endif +} + +constexpr uint32_t Bit(uint32_t index) { + return uint32_t{1} << index; +} + +constexpr bool HasAll(uint32_t value, uint32_t mask) { + return (value & mask) == mask; +} + +JsonifierIsa DetectSupportedIsa() { + if (QueryCpuid(0, 0).eax < 7 || QueryCpuid(0x80000000, 0).eax < 0x80000001) { + return JsonifierIsa::kFallback; + } + constexpr uint32_t kPclmulPopcntOsxsaveAvx = + Bit(1) | Bit(23) | Bit(27) | Bit(28); + constexpr uint32_t kBmi1Avx2Bmi2 = Bit(3) | Bit(5) | Bit(8); + constexpr uint32_t kLzcnt = Bit(5); + constexpr uint64_t kXmmYmmState = 0x6; + const CpuidRegisters leaf1 = QueryCpuid(1, 0); + if (!HasAll(leaf1.ecx, kPclmulPopcntOsxsaveAvx)) { + return JsonifierIsa::kFallback; + } + const uint64_t xcr0 = QueryXcr0(); + const CpuidRegisters leaf7 = QueryCpuid(7, 0); + if ((xcr0 & kXmmYmmState) != kXmmYmmState || + !HasAll(leaf7.ebx, kBmi1Avx2Bmi2) || + !HasAll(QueryCpuid(0x80000001, 0).ecx, kLzcnt)) { + return JsonifierIsa::kFallback; + } + constexpr uint32_t kAvx512FDqBwVl = Bit(16) | Bit(17) | Bit(30) | Bit(31); + constexpr uint32_t kAvx512Vbmi2 = Bit(6); + constexpr uint64_t kOpmaskZmmState = 0xE6; + if ((xcr0 & kOpmaskZmmState) == kOpmaskZmmState && + HasAll(leaf7.ebx, kAvx512FDqBwVl) && HasAll(leaf7.ecx, kAvx512Vbmi2)) { + return JsonifierIsa::kAvx512; + } + return JsonifierIsa::kAvx2; +} + +JsonifierIsa RequestedIsaCap() { + const char* requested = std::getenv("NODE_JSONIFIER_ISA"); + if (requested == nullptr) { + return JsonifierIsa::kAvx512; + } + const std::string_view name{requested}; + if (name == "fallback") { + return JsonifierIsa::kFallback; + } + if (name == "avx2") { + return JsonifierIsa::kAvx2; + } + return JsonifierIsa::kAvx512; +} + +JsonifierIsa DetectIsa() { + return std::min(DetectSupportedIsa(), RequestedIsaCap()); +} + +#elif NODE_JSONIFIER_ARM64 + +JsonifierIsa DetectIsa() { + return JsonifierIsa::kNeon; +} + +#else + +JsonifierIsa DetectIsa() { + return JsonifierIsa::kFallback; +} + +#endif + +JsonifierIsa ActiveIsa() { + static const JsonifierIsa isa = DetectIsa(); + return isa; +} + +PackageJSONBackendParser* NewGenericParser() { + switch (ActiveIsa()) { +#if NODE_JSONIFIER_X64 + case JsonifierIsa::kAvx512: + return isa_avx512::NewGenericParser(); + case JsonifierIsa::kAvx2: + return isa_avx2::NewGenericParser(); + default: + return isa_fallback::NewGenericParser(); +#elif NODE_JSONIFIER_ARM64 + default: + return isa_neon::NewGenericParser(); +#else + default: + return isa_fallback::NewGenericParser(); +#endif + } +} + +std::string_view WalkJsonForIsa(std::string_view json, + json::JsonEventSink* sink) { + switch (ActiveIsa()) { +#if NODE_JSONIFIER_X64 + case JsonifierIsa::kAvx512: + return isa_avx512::WalkJson(json, sink); + case JsonifierIsa::kAvx2: + return isa_avx2::WalkJson(json, sink); + default: + return isa_fallback::WalkJson(json, sink); +#elif NODE_JSONIFIER_ARM64 + default: + return isa_neon::WalkJson(json, sink); +#else + default: + return isa_fallback::WalkJson(json, sink); +#endif + } +} + +} // namespace + +PackageJSONBackendParser::~PackageJSONBackendParser() = default; + +void PackageJSONFieldSink::SetName(std::string_view value) { + fields_->name.emplace(value); +} + +void PackageJSONFieldSink::SetMain(std::string_view value) { + fields_->main.emplace(value); +} + +void PackageJSONFieldSink::SetType(std::string_view value) { + fields_->type.assign(value); +} + +void PackageJSONFieldSink::SetExports(std::string_view value) { + fields_->exports.emplace(value); +} + +void PackageJSONFieldSink::SetImports(std::string_view value) { + fields_->imports.emplace(value); +} + +void PackageJSONFieldSink::SetScripts(std::string_view value) { + fields_->scripts.emplace(value); +} + +void PackageJSONFieldSink::SetPrivate(bool value) { + fields_->is_private = value; +} + +struct JsonifierPackageJSONParser::Impl { + std::unique_ptr backend{NewGenericParser()}; +}; + +JsonifierPackageJSONParser::JsonifierPackageJSONParser() + : impl_(std::make_unique()) {} + +JsonifierPackageJSONParser::~JsonifierPackageJSONParser() = default; + +bool JsonifierPackageJSONParser::Parse(std::string_view json, + PackageJSONFields* fields) { + if (!EndsWithObjectClose(json)) { + return false; + } + PackageJSONFieldSink sink{fields}; + return impl_->backend->Parse(json, &sink); +} + +} // namespace modules +} // namespace node + +namespace node { +namespace json { + +JsonEventSink::~JsonEventSink() = default; + +std::string_view WalkJson(std::string_view json, JsonEventSink* sink) { + return modules::WalkJsonForIsa(json, sink); +} + +} // namespace json +} // namespace node diff --git a/deps/jsonifier/package_json_parser_avx2.cc b/deps/jsonifier/package_json_parser_avx2.cc new file mode 100644 index 000000000000..ae6b00c7fa07 --- /dev/null +++ b/deps/jsonifier/package_json_parser_avx2.cc @@ -0,0 +1,9 @@ +#define NODE_JSONIFIER_ISA_AVX2 1 + +#include "jsonifier_isa_begin.h" + +#include "package_json_generic_parser-inl.h" + +#include "json_walker-inl.h" + +#include "jsonifier_isa_end.h" diff --git a/deps/jsonifier/package_json_parser_avx512.cc b/deps/jsonifier/package_json_parser_avx512.cc new file mode 100644 index 000000000000..c421cfeb176a --- /dev/null +++ b/deps/jsonifier/package_json_parser_avx512.cc @@ -0,0 +1,9 @@ +#define NODE_JSONIFIER_ISA_AVX512 1 + +#include "jsonifier_isa_begin.h" + +#include "package_json_generic_parser-inl.h" + +#include "json_walker-inl.h" + +#include "jsonifier_isa_end.h" diff --git a/deps/jsonifier/package_json_parser_fallback.cc b/deps/jsonifier/package_json_parser_fallback.cc new file mode 100644 index 000000000000..8a167365db76 --- /dev/null +++ b/deps/jsonifier/package_json_parser_fallback.cc @@ -0,0 +1,9 @@ +#define NODE_JSONIFIER_ISA_FALLBACK 1 + +#include "jsonifier_isa_begin.h" + +#include "package_json_generic_parser-inl.h" + +#include "json_walker-inl.h" + +#include "jsonifier_isa_end.h" diff --git a/deps/jsonifier/package_json_parser_neon.cc b/deps/jsonifier/package_json_parser_neon.cc new file mode 100644 index 000000000000..ce34ac9b5ffa --- /dev/null +++ b/deps/jsonifier/package_json_parser_neon.cc @@ -0,0 +1,9 @@ +#define NODE_JSONIFIER_ISA_NEON 1 + +#include "jsonifier_isa_begin.h" + +#include "package_json_generic_parser-inl.h" + +#include "json_walker-inl.h" + +#include "jsonifier_isa_end.h" diff --git a/deps/simdjson/BUILD.gn b/deps/simdjson/BUILD.gn deleted file mode 100644 index d0580ccf354d..000000000000 --- a/deps/simdjson/BUILD.gn +++ /dev/null @@ -1,14 +0,0 @@ -############################################################################## -# # -# DO NOT EDIT THIS FILE! # -# # -############################################################################## - -# This file is used by GN for building, which is NOT the build system used for -# building official binaries. -# Please modify the gyp files if you are making changes to build system. - -import("unofficial.gni") - -simdjson_gn_build("simdjson") { -} diff --git a/deps/simdjson/LICENSE b/deps/simdjson/LICENSE deleted file mode 100644 index 57fc54cc1f77..000000000000 --- a/deps/simdjson/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2018-2025 The simdjson authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/deps/simdjson/simdjson.cpp b/deps/simdjson/simdjson.cpp deleted file mode 100644 index 6db84f295573..000000000000 --- a/deps/simdjson/simdjson.cpp +++ /dev/null @@ -1,79836 +0,0 @@ -/* auto-generated on 2026-10-31 19:14:12 -0400. version 5.1.1 Do not edit! */ -/* including simdjson.cpp: */ -/* begin file simdjson.cpp */ -#define SIMDJSON_SRC_SIMDJSON_CPP - -/* including base.h: #include */ -/* begin file base.h */ -#ifndef SIMDJSON_SRC_BASE_H -#define SIMDJSON_SRC_BASE_H - -/* including simdjson/base.h: #include */ -/* begin file simdjson/base.h */ -/** - * @file Base declarations for all simdjson headers - * @private - */ -#ifndef SIMDJSON_BASE_H -#define SIMDJSON_BASE_H - -/* including simdjson/common_defs.h: #include "simdjson/common_defs.h" */ -/* begin file simdjson/common_defs.h */ -#ifndef SIMDJSON_COMMON_DEFS_H -#define SIMDJSON_COMMON_DEFS_H - -#include -/* including simdjson/compiler_check.h: #include "simdjson/compiler_check.h" */ -/* begin file simdjson/compiler_check.h */ -#ifndef SIMDJSON_COMPILER_CHECK_H -#define SIMDJSON_COMPILER_CHECK_H - -#ifndef __cplusplus -#error simdjson requires a C++ compiler -#endif - -#ifndef SIMDJSON_CPLUSPLUS -#if defined(_MSVC_LANG) && !defined(__clang__) -#define SIMDJSON_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG) -#else -#define SIMDJSON_CPLUSPLUS __cplusplus -#endif -#endif - -// C++ 26 -// While C++26 is a working draft, compilers report 202400L in C++26 mode -// (both GCC 16 and Clang 21 do). Update when the standard is finalized. -#if !defined(SIMDJSON_CPLUSPLUS26) && (SIMDJSON_CPLUSPLUS >= 202400L) -#define SIMDJSON_CPLUSPLUS26 1 -#endif - -// C++ 23 -#if !defined(SIMDJSON_CPLUSPLUS23) && (SIMDJSON_CPLUSPLUS >= 202302L) -#define SIMDJSON_CPLUSPLUS23 1 -#endif - -// C++ 20 -#if !defined(SIMDJSON_CPLUSPLUS20) && (SIMDJSON_CPLUSPLUS >= 202002L) -#define SIMDJSON_CPLUSPLUS20 1 -#endif - -// C++ 17 -#if !defined(SIMDJSON_CPLUSPLUS17) && (SIMDJSON_CPLUSPLUS >= 201703L) -#define SIMDJSON_CPLUSPLUS17 1 -#endif - -// C++ 14 -#if !defined(SIMDJSON_CPLUSPLUS14) && (SIMDJSON_CPLUSPLUS >= 201402L) -#define SIMDJSON_CPLUSPLUS14 1 -#endif - -// C++ 11 -#if !defined(SIMDJSON_CPLUSPLUS11) && (SIMDJSON_CPLUSPLUS >= 201103L) -#define SIMDJSON_CPLUSPLUS11 1 -#endif - -#ifndef SIMDJSON_CPLUSPLUS11 -#error simdjson requires a compiler compliant with the C++11 standard -#endif - -#ifndef SIMDJSON_IF_CONSTEXPR -#if SIMDJSON_CPLUSPLUS17 -#define SIMDJSON_IF_CONSTEXPR if constexpr -#else -#define SIMDJSON_IF_CONSTEXPR if -#endif -#endif - -#ifndef SIMDJSON_CONSTEXPR_LAMBDA -#if SIMDJSON_CPLUSPLUS17 -#define SIMDJSON_CONSTEXPR_LAMBDA constexpr -#else -#define SIMDJSON_CONSTEXPR_LAMBDA -#endif -#endif - - - -#ifdef __has_include -#if __has_include() -#include -#endif -#endif - -// Static reflection. -// -// The reflection-based APIs (simdjson::to, document::get, the builder, -// compile-time JSON, annotations) need considerably more than the reflection -// operator. We turn them on only when the compiler advertises all of: -// -// P2996 reflection (^^, splicers, ) __cpp_impl_reflection, -// __cpp_lib_reflection -// P1306 expansion statements (template for) __cpp_expansion_statements -// P3491 std::define_static_string / _array __cpp_lib_define_static -// -// Two further features we rely on have, as of this writing, no feature-test -// macro of their own, so they cannot be checked directly: -// -// P3394 annotations ([[=x]], std::meta::annotations_of) -- used for -// the annotations of simdjson/annotations.h (rename, skip, ...). -// P3289 consteval blocks (consteval { ... }) -- used by compile_time_json. -// -// Every implementation that defines the four macros above also implements -// those two, so requiring the four is sufficient in practice. If that ever -// stops being true, define SIMDJSON_STATIC_REFLECTION=0 to opt out. -// -// SIMDJSON_STATIC_REFLECTION may always be defined by the user (or by the -// build system) to 0 or 1 to override the detection. -// -// Note that C++26 mode alone is not enough: GCC 16 requires -freflection, -// and only then does it define __cpp_impl_reflection. -#ifndef SIMDJSON_STATIC_REFLECTION -#if defined(SIMDJSON_CPLUSPLUS26) && \ - defined(__cpp_impl_reflection) && __cpp_impl_reflection >= 202506L && \ - defined(__cpp_lib_reflection) && __cpp_lib_reflection >= 202506L && \ - defined(__cpp_expansion_statements) && \ - __cpp_expansion_statements >= 202506L && \ - defined(__cpp_lib_define_static) && __cpp_lib_define_static >= 202506L -// __cpp_lib_reflection is the feature-test macro for , so there is no -// need for a separate __has_include check (which would have to be guarded for -// compilers that lack __has_include). -#define SIMDJSON_STATIC_REFLECTION 1 -#else -#define SIMDJSON_STATIC_REFLECTION 0 -#endif -#endif // SIMDJSON_STATIC_REFLECTION - -#if defined(__apple_build_version__) -#if __apple_build_version__ < 14000000 -#define SIMDJSON_CONCEPT_DISABLED 1 // apple-clang/13 doesn't support std::convertible_to -#endif -#endif - -#if defined(__cpp_lib_ranges) && __cpp_lib_ranges >= 201911L -#include -#define SIMDJSON_SUPPORTS_RANGES 1 -#else -#define SIMDJSON_SUPPORTS_RANGES 0 -#endif - -#if defined(__cpp_concepts) && !defined(SIMDJSON_CONCEPT_DISABLED) -#if __cpp_concepts >= 201907L -#include -#define SIMDJSON_SUPPORTS_CONCEPTS 1 -#else -#define SIMDJSON_SUPPORTS_CONCEPTS 0 -#endif -#else // defined(__cpp_concepts) && !defined(SIMDJSON_CONCEPT_DISABLED) -#define SIMDJSON_SUPPORTS_CONCEPTS 0 -#endif // defined(__cpp_concepts) && !defined(SIMDJSON_CONCEPT_DISABLED) - -// copy SIMDJSON_SUPPORTS_CONCEPTS to SIMDJSON_SUPPORTS_DESERIALIZATION. -#if SIMDJSON_SUPPORTS_CONCEPTS -#define SIMDJSON_SUPPORTS_DESERIALIZATION 1 -#else -#define SIMDJSON_SUPPORTS_DESERIALIZATION 0 -#endif - -// The C++20 char8_t type (and std::u8string/std::u8string_view) is available. -// Because all strings that simdjson produces are valid UTF-8, we can offer -// char8_t variants of our string accessors when this macro is set. -#if !defined(SIMDJSON_SUPPORTS_CHAR8_T) -#if defined(__cpp_char8_t) && __cpp_char8_t >= 201811L -#define SIMDJSON_SUPPORTS_CHAR8_T 1 -#else -#define SIMDJSON_SUPPORTS_CHAR8_T 0 -#endif -#endif // !defined(SIMDJSON_SUPPORTS_CHAR8_T) - -// The C++23 fixed-width floating-point types std::float32_t and std::float64_t -// () are available. They are optional even in C++23: a compiler that -// provides them predefines __STDCPP_FLOAT32_T__ and __STDCPP_FLOAT64_T__. -// When these macros are set, we offer get_float32() and get_float64(). -#if !defined(SIMDJSON_SUPPORTS_FLOAT32_T) -#if defined(__STDCPP_FLOAT32_T__) && defined(__has_include) -#if __has_include() -#define SIMDJSON_SUPPORTS_FLOAT32_T 1 -#endif -#endif -#ifndef SIMDJSON_SUPPORTS_FLOAT32_T -#define SIMDJSON_SUPPORTS_FLOAT32_T 0 -#endif -#endif // !defined(SIMDJSON_SUPPORTS_FLOAT32_T) - -#if !defined(SIMDJSON_SUPPORTS_FLOAT64_T) -#if defined(__STDCPP_FLOAT64_T__) && defined(__has_include) -#if __has_include() -#define SIMDJSON_SUPPORTS_FLOAT64_T 1 -#endif -#endif -#ifndef SIMDJSON_SUPPORTS_FLOAT64_T -#define SIMDJSON_SUPPORTS_FLOAT64_T 0 -#endif -#endif // !defined(SIMDJSON_SUPPORTS_FLOAT64_T) - -#if SIMDJSON_SUPPORTS_FLOAT32_T || SIMDJSON_SUPPORTS_FLOAT64_T -#include -#endif - - -#if !defined(SIMDJSON_CONSTEVAL) -#if defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L -#define SIMDJSON_CONSTEVAL 1 -#else -#define SIMDJSON_CONSTEVAL 0 -#endif // defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L -#endif // !defined(SIMDJSON_CONSTEVAL) - -// SIMDJSON_CONSTEXPR_STRING is 'constexpr' when the standard library supports -// constexpr std::string (e.g., libstdc++ 12 or better), and empty otherwise. It -// lets functions that build a std::string be constant expressions when possible -// while still compiling against older standard libraries. -#if !defined(SIMDJSON_CONSTEXPR_STRING) -#if defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L -#define SIMDJSON_CONSTEXPR_STRING constexpr -#else -#define SIMDJSON_CONSTEXPR_STRING -#endif // defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L -#endif // !defined(SIMDJSON_CONSTEXPR_STRING) -#endif // SIMDJSON_COMPILER_CHECK_H -/* end file simdjson/compiler_check.h */ -/* including simdjson/portability.h: #include "simdjson/portability.h" */ -/* begin file simdjson/portability.h */ -#ifndef SIMDJSON_PORTABILITY_H -#define SIMDJSON_PORTABILITY_H - -#include -#include -#include -#include -#include -#include -#ifndef _WIN32 -// strcasecmp, strncasecmp -#include -#endif - -static_assert(CHAR_BIT == 8, "simdjson requires 8-bit bytes"); - - -// We are using size_t without namespace std:: throughout the project -using std::size_t; - -#ifdef _MSC_VER -#define SIMDJSON_VISUAL_STUDIO 1 -/** - * We want to differentiate carefully between - * clang under visual studio and regular visual - * studio. - * - * Under clang for Windows, we enable: - * * target pragmas so that part and only part of the - * code gets compiled for advanced instructions. - * - */ -#ifdef __clang__ -// clang under visual studio -#define SIMDJSON_CLANG_VISUAL_STUDIO 1 -#else -// just regular visual studio (best guess) -#define SIMDJSON_REGULAR_VISUAL_STUDIO 1 -#endif // __clang__ -#endif // _MSC_VER - -#if (defined(__x86_64__) || defined(_M_AMD64)) && !defined(_M_ARM64EC) -#define SIMDJSON_IS_X86_64 1 -#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) -#define SIMDJSON_IS_ARM64 1 -#elif defined(__riscv) && __riscv_xlen == 64 -#define SIMDJSON_IS_RISCV64 1 - - #if __riscv_v_intrinsic >= 11000 - #define SIMDJSON_HAS_RVV_INTRINSICS 1 - #endif - - #if SIMDJSON_HAS_RVV_INTRINSICS && __riscv_vector && __riscv_v_min_vlen >= 128 && __riscv_v_elen >= 64 - #define SIMDJSON_IS_RVV 1 // RISC-V V extension - #endif - - // current toolchains don't support fixed-size SIMD types that don't match VLEN directly - #if __riscv_v_fixed_vlen >= 128 && __riscv_v_fixed_vlen <= 512 - #define SIMDJSON_IS_RVV_VLS 1 - #endif - -#elif defined(__loongarch_lp64) -#define SIMDJSON_IS_LOONGARCH64 1 -#if defined(__loongarch_sx) && defined(__loongarch_asx) - #define SIMDJSON_IS_LSX 1 - #define SIMDJSON_IS_LASX 1 // We can always run both -#elif defined(__loongarch_sx) - #define SIMDJSON_IS_LSX 1 - -// Adjust for runtime dispatching support. -#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) && !defined(__NVCOMPILER) -#if __GNUC__ > 15 || (__GNUC__ == 15 && __GNUC_MINOR__ >= 0) - // We are ok, we will support runtime dispatch for LASX. -#else - // We disable runtime dispatch for LASX, which means that we will not be able to use LASX - // even if it is supported by the hardware. - // Loongson users should update to GCC 15 or better. - #define SIMDJSON_IMPLEMENTATION_LASX 0 -#endif -#else - // We are not using GCC, so we assume that we can support runtime dispatch for LASX. - // https://godbolt.org/z/jcMnrjYhs - #define SIMDJSON_IMPLEMENTATION_LASX 0 -#endif - - - -#endif -#elif defined(__PPC64__) || defined(_M_PPC64) -#define SIMDJSON_IS_PPC64 1 -#if defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) -#define SIMDJSON_IS_PPC64_VMX 1 -#endif // defined(__ALTIVEC__) -#else -#define SIMDJSON_IS_32BITS 1 - -#if defined(_M_IX86) || defined(__i386__) -#define SIMDJSON_IS_X86_32BITS 1 -#elif defined(__arm__) || defined(_M_ARM) -#define SIMDJSON_IS_ARM_32BITS 1 -#elif defined(__PPC__) || defined(_M_PPC) -#define SIMDJSON_IS_PPC_32BITS 1 -#endif - -#endif // defined(__x86_64__) || defined(_M_AMD64) -#ifndef SIMDJSON_IS_32BITS -#define SIMDJSON_IS_32BITS 0 -#endif - -#if SIMDJSON_IS_32BITS -#ifndef SIMDJSON_NO_PORTABILITY_WARNING -// In the future, we should allow programmers -// to get warning. -#endif // SIMDJSON_NO_PORTABILITY_WARNING -#endif // SIMDJSON_IS_32BITS - -#define SIMDJSON_CAT_IMPLEMENTATION_(a,...) a ## __VA_ARGS__ -#define SIMDJSON_CAT(a,...) SIMDJSON_CAT_IMPLEMENTATION_(a, __VA_ARGS__) - -#define SIMDJSON_STRINGIFY_IMPLEMENTATION_(a,...) #a SIMDJSON_STRINGIFY(__VA_ARGS__) -#define SIMDJSON_STRINGIFY(a,...) SIMDJSON_CAT_IMPLEMENTATION_(a, __VA_ARGS__) - -// this is almost standard? -#undef SIMDJSON_STRINGIFY_IMPLEMENTATION_ -#undef SIMDJSON_STRINGIFY -#define SIMDJSON_STRINGIFY_IMPLEMENTATION_(a) #a -#define SIMDJSON_STRINGIFY(a) SIMDJSON_STRINGIFY_IMPLEMENTATION_(a) - -// Our fast kernels require 64-bit systems. -// -// On 32-bit x86, we lack 64-bit popcnt, lzcnt, blsr instructions. -// Furthermore, the number of SIMD registers is reduced. -// -// On 32-bit ARM, we would have smaller registers. -// -// The simdjson users should still have the fallback kernel. It is -// slower, but it should run everywhere. - -// -// Enable valid runtime implementations, and select SIMDJSON_BUILTIN_IMPLEMENTATION -// - -// We are going to use runtime dispatch. -#if defined(SIMDJSON_IS_X86_64) || defined(SIMDJSON_IS_LSX) -#ifdef __clang__ -// clang does not have GCC push pop -// warning: clang attribute push can't be used within a namespace in clang up -// til 8.0 so SIMDJSON_TARGET_REGION and SIMDJSON_UNTARGET_REGION must be *outside* of a -// namespace. -#define SIMDJSON_TARGET_REGION(T) \ - _Pragma(SIMDJSON_STRINGIFY( \ - clang attribute push(__attribute__((target(T))), apply_to = function))) -#define SIMDJSON_UNTARGET_REGION _Pragma("clang attribute pop") -#elif defined(__GNUC__) -// GCC is easier -#define SIMDJSON_TARGET_REGION(T) \ - _Pragma("GCC push_options") _Pragma(SIMDJSON_STRINGIFY(GCC target(T))) -#define SIMDJSON_UNTARGET_REGION _Pragma("GCC pop_options") -#endif // clang then gcc - -#endif // defined(SIMDJSON_IS_X86_64) || defined(SIMDJSON_IS_LSX) - -// Default target region macros don't do anything. -#ifndef SIMDJSON_TARGET_REGION -#define SIMDJSON_TARGET_REGION(T) -#define SIMDJSON_UNTARGET_REGION -#endif - -// Is threading enabled? -#if defined(_REENTRANT) || defined(_MT) -#ifndef SIMDJSON_THREADS_ENABLED -#define SIMDJSON_THREADS_ENABLED -#endif -#endif - -// workaround for large stack sizes under -O0. -// https://github.com/simdjson/simdjson/issues/691 -#ifdef __APPLE__ -#ifndef __OPTIMIZE__ -// Apple systems have small stack sizes in secondary threads. -// Lack of compiler optimization may generate high stack usage. -// Users may want to disable threads for safety, but only when -// in debug mode which we detect by the fact that the __OPTIMIZE__ -// macro is not defined. -#undef SIMDJSON_THREADS_ENABLED -#endif -#endif - - -#if defined(__clang__) -#define SIMDJSON_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined"))) -#elif defined(__GNUC__) -#define SIMDJSON_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined)) -#else -#define SIMDJSON_NO_SANITIZE_UNDEFINED -#endif - -#if defined(__clang__) || defined(__GNUC__) -#define simdjson_pure [[gnu::pure]] -#else -#define simdjson_pure -#endif - -#if defined(__clang__) || defined(__GNUC__) -#if defined(__has_feature) -# if __has_feature(memory_sanitizer) -#define SIMDJSON_NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory"))) -# endif // if __has_feature(memory_sanitizer) -#endif // defined(__has_feature) -#endif -// make sure it is defined as 'nothing' if it is unapplicable. -#ifndef SIMDJSON_NO_SANITIZE_MEMORY -#define SIMDJSON_NO_SANITIZE_MEMORY -#endif - -#if SIMDJSON_VISUAL_STUDIO -// This is one case where we do not distinguish between -// regular visual studio and clang under visual studio. -// clang under Windows has _stricmp (like visual studio) but not strcasecmp (as clang normally has) -#define simdjson_strcasecmp _stricmp -#define simdjson_strncasecmp _strnicmp -#else -// The strcasecmp, strncasecmp, and strcasestr functions do not work with multibyte strings (e.g. UTF-8). -// So they are only useful for ASCII in our context. -// https://www.gnu.org/software/libunistring/manual/libunistring.html#char-_002a-strings -#define simdjson_strcasecmp strcasecmp -#define simdjson_strncasecmp strncasecmp -#endif - -#if (defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG))) && !SIMDJSON_DEVELOPMENT_CHECKS -// If SIMDJSON_DEVELOPMENT_CHECKS is undefined or 0, we consider that we are in release mode. -// If NDEBUG is set, or __OPTIMIZE__ is set, or we are under MSVC in release mode, -// then do away with asserts and use __assume. -// We still recommend that our users set NDEBUG in release mode. -#if SIMDJSON_VISUAL_STUDIO -#define SIMDJSON_UNREACHABLE() __assume(0) -#define SIMDJSON_ASSUME(COND) __assume(COND) -#else -#define SIMDJSON_UNREACHABLE() __builtin_unreachable(); -#define SIMDJSON_ASSUME(COND) do { if (!(COND)) __builtin_unreachable(); } while (0) -#endif - -#else // defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG)) && !SIMDJSON_DEVELOPMENT_CHECKS -// This should only ever be enabled in debug mode. -#define SIMDJSON_UNREACHABLE() assert(0); -#define SIMDJSON_ASSUME(COND) assert(COND) - -#endif - - - -#if defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__ -#define SIMDJSON_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) -#elif defined _WIN32 -#define SIMDJSON_IS_BIG_ENDIAN 0 -#else -#if defined(__APPLE__) || defined(__FreeBSD__) -#include -#elif defined(sun) || defined(__sun) -#include -#elif defined(__MVS__) -#include -#else -#ifdef __has_include -#if __has_include() -#include -#endif //__has_include() -#endif //__has_include -#endif -# -#ifndef __BYTE_ORDER__ -// safe choice -#define SIMDJSON_IS_BIG_ENDIAN 0 -#endif -# -#ifndef __ORDER_LITTLE_ENDIAN__ -// safe choice -#define SIMDJSON_IS_BIG_ENDIAN 0 -#endif -# -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define SIMDJSON_IS_BIG_ENDIAN 0 -#else -#define SIMDJSON_IS_BIG_ENDIAN 1 -#endif -#endif - -#ifndef SIMDJSON_HAS_UNISTD_H -#if defined(__unix__) || defined(__APPLE__) || defined(__linux__) -#define SIMDJSON_HAS_UNISTD_H 1 -#else -#define SIMDJSON_HAS_UNISTD_H 0 -#endif -#endif - -// padded_memory_map availability. -// -// On POSIX platforms the class is always available: the implementation uses -// `mmap` (and a trailing anonymous page for padding) from . -// -// On Windows the class is disabled by default and must be explicitly -// opted into by defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`. Enabling -// it requires: -// 1. `` has been included *before* `` (so that -// this header can see the Win32 types and the `_WINDOWS_` include -// guard), -// 2. the compilation targets Windows 10, version 1803 or later -// (i.e. `NTDDI_VERSION >= NTDDI_WIN10_RS4`, `0x0A000005`). This is -// required because the implementation relies on the modern memory -// APIs introduced with that version (`CreateFileMapping2` / -// `MapViewOfFile3`), -// 3. the link step pulls in an import library that exports those APIs, -// typically `onecore.lib` (or `mincore.lib`). -// -// The `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS` CMake option arranges (1)-(3) -// automatically when building simdjson with its own CMake. Consumers using -// simdjson as a pre-built library are responsible for setting the macro, -// the Windows version macros, and the link library themselves. -// -// If the opt-in conditions are not met on Windows, `padded_memory_map` -// simply does not exist -- any attempt to use it fails at compile time -// with an "unknown identifier" diagnostic rather than silently degrading. -// -// The SIMDJSON_HAS_PADDED_MEMORY_MAP macro reflects whether the class is -// available in the current translation unit. Users may test this macro to -// conditionally compile code that depends on padded_memory_map. -#ifndef SIMDJSON_HAS_PADDED_MEMORY_MAP - #if defined(__unix__) || defined(__APPLE__) || defined(__linux__) - #define SIMDJSON_HAS_PADDED_MEMORY_MAP 1 - #elif defined(_WINDOWS_) && defined(SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS) && SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS - #define SIMDJSON_HAS_PADDED_MEMORY_MAP 1 - #else - #define SIMDJSON_HAS_PADDED_MEMORY_MAP 0 - #endif -#endif - -#endif // SIMDJSON_PORTABILITY_H -/* end file simdjson/portability.h */ -#include - -namespace simdjson { -namespace internal { -/** - * @private - * Scratch capacity that every caller of to_chars must provide. - * - * The emitted decimal is at most ~24 characters, but dragonbox() and - * format_buffer() intentionally write past the logical end with fixed-size - * 16/17-byte memcpy/memset operations so the compiler can inline them (no - * libc mem* dispatch with size-class branches). The extra bytes are required - * for safety of those over-writes; do not shrink this below 40. - * See src/to_chars.cpp and #2805. - */ -// Use an unscoped enum (not static constexpr / inline constexpr): -// - C++11 targets (readme_examples11, quickstart11, ...) still include this header -// - a static constexpr in the amalgamated simdjson.cpp TU is unused there -// (only callers in headers use it) and trips -Wunused-const-variable -Werror -enum : size_t { to_chars_buffer_size = 40 }; -/** - * @private - * Our own implementation of the C++17 to_chars function. - * Defined in src/to_chars - * - * @note The buffer starting at first must have at least to_chars_buffer_size - * bytes of writable storage (see to_chars_buffer_size). - * @note The input number must be finite (NaN/Inf are not supported). - * @note The result is NOT null-terminated. - */ -char *to_chars(char *first, const char *last, double value); -/** - * @private - * A number parsing routine. - * Defined in src/from_chars - */ -double from_chars(const char *first) noexcept; -double from_chars(const char *first, const char* end) noexcept; -/** - * @private - * Same as from_chars, but produces a correctly rounded binary32 (float) value. - * Defined in src/from_chars - */ -float from_chars_float(const char *first) noexcept; -} - -#ifndef SIMDJSON_EXCEPTIONS -#if defined(__cpp_exceptions) || defined(_CPPUNWIND) -#define SIMDJSON_EXCEPTIONS 1 -#else -#define SIMDJSON_EXCEPTIONS 0 -#endif -#endif - -#ifndef SIMDJSON_ENABLE_NAN_INF -#define SIMDJSON_ENABLE_NAN_INF 0 -#endif - -} // namespace simdjson - -#if defined(__GNUC__) - // Marks a block with a name so that MCA analysis can see it. - #define SIMDJSON_BEGIN_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-BEGIN " #name); - #define SIMDJSON_END_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-END " #name); - #define SIMDJSON_DEBUG_BLOCK(name, block) BEGIN_DEBUG_BLOCK(name); block; END_DEBUG_BLOCK(name); -#else - #define SIMDJSON_BEGIN_DEBUG_BLOCK(name) - #define SIMDJSON_END_DEBUG_BLOCK(name) - #define SIMDJSON_DEBUG_BLOCK(name, block) -#endif - -// Align to N-byte boundary -#define SIMDJSON_ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1)) - -#if SIMDJSON_REGULAR_VISUAL_STUDIO - // We could use [[deprecated]] but it requires C++14 - #define simdjson_deprecated __declspec(deprecated) - - #define simdjson_really_inline __forceinline - #define simdjson_never_inline inline __declspec(noinline) - #define simdjson_really_flatten [[msvc::flatten]] - - #define simdjson_unused - #define simdjson_warn_unused - - #ifndef simdjson_likely - #define simdjson_likely(x) x - #endif - #ifndef simdjson_unlikely - #define simdjson_unlikely(x) x - #endif - - #define SIMDJSON_PUSH_DISABLE_WARNINGS __pragma(warning( push )) - #define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS __pragma(warning( push, 0 )) - #define SIMDJSON_DISABLE_VS_WARNING(WARNING_NUMBER) __pragma(warning( disable : WARNING_NUMBER )) - // Get rid of Intellisense-only warnings (Code Analysis) - // Though __has_include is C++17, it is supported in Visual Studio 2017 or better (_MSC_VER>=1910). - #ifdef __has_include - #if __has_include() - #include - #define SIMDJSON_DISABLE_UNDESIRED_WARNINGS SIMDJSON_DISABLE_VS_WARNING(ALL_CPPCORECHECK_WARNINGS) - #endif - #endif - - #ifndef SIMDJSON_DISABLE_UNDESIRED_WARNINGS - #define SIMDJSON_DISABLE_UNDESIRED_WARNINGS - #endif - - #define SIMDJSON_DISABLE_DEPRECATED_WARNING SIMDJSON_DISABLE_VS_WARNING(4996) - #define SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING - #define SIMDJSON_POP_DISABLE_WARNINGS __pragma(warning( pop )) - - #define SIMDJSON_PUSH_DISABLE_UNUSED_WARNINGS - #define SIMDJSON_POP_DISABLE_UNUSED_WARNINGS - -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - // We could use [[deprecated]] but it requires C++14 - #define simdjson_deprecated __attribute__((deprecated)) - - #define simdjson_really_inline inline __attribute__((always_inline)) - #define simdjson_never_inline inline __attribute__((noinline)) - #define simdjson_really_flatten [[gnu::flatten]] - - #define simdjson_unused __attribute__((unused)) - #define simdjson_warn_unused __attribute__((warn_unused_result)) - - #ifndef simdjson_likely - #define simdjson_likely(x) __builtin_expect(!!(x), 1) - #endif - #ifndef simdjson_unlikely - #define simdjson_unlikely(x) __builtin_expect(!!(x), 0) - #endif - - #define SIMDJSON_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push") - // gcc doesn't seem to disable all warnings with all and extra, add warnings here as necessary - // We do it separately for clang since it has different warnings. - #ifdef __clang__ - // clang is missing -Wmaybe-uninitialized. - #define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \ - SIMDJSON_DISABLE_GCC_WARNING(-Weffc++) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wall) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wconversion) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wextra) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wattributes) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wreturn-type) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wshadow) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wunused-parameter) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wunused-variable) - #else // __clang__ - #define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \ - SIMDJSON_DISABLE_GCC_WARNING(-Weffc++) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wall) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wconversion) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wextra) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wattributes) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wreturn-type) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wshadow) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wunused-parameter) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wunused-variable) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wmaybe-uninitialized) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wformat-security) - #endif // __clang__ - - #define SIMDJSON_PRAGMA(P) _Pragma(#P) - #define SIMDJSON_DISABLE_GCC_WARNING(WARNING) SIMDJSON_PRAGMA(GCC diagnostic ignored #WARNING) - #if SIMDJSON_CLANG_VISUAL_STUDIO - #define SIMDJSON_DISABLE_UNDESIRED_WARNINGS SIMDJSON_DISABLE_GCC_WARNING(-Wmicrosoft-include) - #else - #define SIMDJSON_DISABLE_UNDESIRED_WARNINGS - #endif - #define SIMDJSON_DISABLE_DEPRECATED_WARNING SIMDJSON_DISABLE_GCC_WARNING(-Wdeprecated-declarations) - #define SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING SIMDJSON_DISABLE_GCC_WARNING(-Wstrict-overflow) - #define SIMDJSON_POP_DISABLE_WARNINGS _Pragma("GCC diagnostic pop") - - #define SIMDJSON_PUSH_DISABLE_UNUSED_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \ - SIMDJSON_DISABLE_GCC_WARNING(-Wunused) - #define SIMDJSON_POP_DISABLE_UNUSED_WARNINGS SIMDJSON_POP_DISABLE_WARNINGS - - - -#endif // MSC_VER - -#if defined(simdjson_inline) - // Prefer the user's definition of simdjson_inline; don't define it ourselves. -#elif defined(__GNUC__) && !defined(__OPTIMIZE__) - // If optimizations are disabled, forcing inlining can lead to significant - // code bloat and high compile times. Don't use simdjson_really_inline for - // unoptimized builds. - #define simdjson_inline inline -#else - // Force inlining for most simdjson functions. - #define simdjson_inline simdjson_really_inline -#endif - -#if defined(simdjson_flatten) - // Prefer the user's definition of simdjson_flatten; don't define it ourselves. -#elif (defined(__GNUC__) && !defined(__OPTIMIZE__)) || (defined(_DEBUG) && _MSC_VER ) - // Flattening can lead to significant code bloat and high compile times. Don't use it for unoptimized builds. - #define simdjson_flatten -#else - #define simdjson_flatten simdjson_really_flatten -#endif - -#if SIMDJSON_VISUAL_STUDIO - /** - * Windows users need to do some extra work when building - * or using a dynamic library (DLL). When building, we need - * to set SIMDJSON_DLLIMPORTEXPORT to __declspec(dllexport). - * When *using* the DLL, the user needs to set - * SIMDJSON_DLLIMPORTEXPORT __declspec(dllimport). - * - * Static libraries not need require such work. - * - * It does not matter here whether you are using - * the regular visual studio or clang under visual - * studio, you still need to handle these issues. - * - * Non-Windows systems do not have this complexity. - */ - #if SIMDJSON_BUILDING_WINDOWS_DYNAMIC_LIBRARY - // We set SIMDJSON_BUILDING_WINDOWS_DYNAMIC_LIBRARY when we build a DLL under Windows. - // It should never happen that both SIMDJSON_BUILDING_WINDOWS_DYNAMIC_LIBRARY and - // SIMDJSON_USING_WINDOWS_DYNAMIC_LIBRARY are set. - #define SIMDJSON_DLLIMPORTEXPORT __declspec(dllexport) - #elif SIMDJSON_USING_WINDOWS_DYNAMIC_LIBRARY - // Windows user who call a dynamic library should set SIMDJSON_USING_WINDOWS_DYNAMIC_LIBRARY to 1. - #define SIMDJSON_DLLIMPORTEXPORT __declspec(dllimport) - #else - // We assume by default static linkage - #define SIMDJSON_DLLIMPORTEXPORT - #endif -#else - #define SIMDJSON_DLLIMPORTEXPORT -#endif - -// C++17 requires string_view. -#if SIMDJSON_CPLUSPLUS17 -#define SIMDJSON_HAS_STRING_VIEW -#include // by the standard, this has to be safe. -#endif - -// This macro (__cpp_lib_string_view) has to be defined -// for C++17 and better, but if it is otherwise defined, -// we are going to assume that string_view is available -// even if we do not have C++17 support. -#ifdef __cpp_lib_string_view -#define SIMDJSON_HAS_STRING_VIEW -#include -#endif - -// Some systems have string_view even if we do not have C++17 support, -// and even if __cpp_lib_string_view is undefined, it is the case -// with Apple clang version 11. -// We must handle it. *This is important.* -#ifndef _MSC_VER -#ifndef SIMDJSON_HAS_STRING_VIEW -#if defined __has_include -// do not combine the next #if with the previous one (unsafe) -#if __has_include () -// now it is safe to trigger the include -#include // though the file is there, it does not follow that we got the implementation -#if defined(_LIBCPP_STRING_VIEW) -// Ah! So we under libc++ which under its Library Fundamentals Technical Specification, which preceded C++17, -// included string_view. -// This means that we have string_view *even though* we may not have C++17. -#define SIMDJSON_HAS_STRING_VIEW -#endif // _LIBCPP_STRING_VIEW -#endif // __has_include () -#endif // defined __has_include -#endif // def SIMDJSON_HAS_STRING_VIEW -#endif // def _MSC_VER -// end of complicated but important routine to try to detect string_view. - -// -// Backfill std::string_view using nonstd::string_view on systems where -// we expect that string_view is missing. Important: if we get this wrong, -// we will end up with two string_view definitions and potential trouble. -// That is why we work so hard above to avoid it. -// -#ifndef SIMDJSON_HAS_STRING_VIEW -SIMDJSON_PUSH_DISABLE_ALL_WARNINGS -/* including simdjson/nonstd/string_view.hpp: #include "simdjson/nonstd/string_view.hpp" */ -/* begin file simdjson/nonstd/string_view.hpp */ -// Copyright 2017-2020 by Martin Moene -// -// string-view lite, a C++17-like string_view for C++98 and later. -// For more information see https://github.com/martinmoene/string-view-lite -// -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - - -#ifndef NONSTD_SV_LITE_H_INCLUDED -#define NONSTD_SV_LITE_H_INCLUDED - -#define string_view_lite_MAJOR 1 -#define string_view_lite_MINOR 8 -#define string_view_lite_PATCH 0 - -#define string_view_lite_VERSION nssv_STRINGIFY(string_view_lite_MAJOR) "." nssv_STRINGIFY(string_view_lite_MINOR) "." nssv_STRINGIFY(string_view_lite_PATCH) - -#define nssv_STRINGIFY( x ) nssv_STRINGIFY_( x ) -#define nssv_STRINGIFY_( x ) #x - -// string-view lite configuration: - -#define nssv_STRING_VIEW_DEFAULT 0 -#define nssv_STRING_VIEW_NONSTD 1 -#define nssv_STRING_VIEW_STD 2 - -// tweak header support: - -#ifdef __has_include -# if __has_include() -# include -# endif -#define nssv_HAVE_TWEAK_HEADER 1 -#else -#define nssv_HAVE_TWEAK_HEADER 0 -//# pragma message("string_view.hpp: Note: Tweak header not supported.") -#endif - -// string_view selection and configuration: - -#if !defined( nssv_CONFIG_SELECT_STRING_VIEW ) -# define nssv_CONFIG_SELECT_STRING_VIEW ( nssv_HAVE_STD_STRING_VIEW ? nssv_STRING_VIEW_STD : nssv_STRING_VIEW_NONSTD ) -#endif - -#ifndef nssv_CONFIG_STD_SV_OPERATOR -# define nssv_CONFIG_STD_SV_OPERATOR 0 -#endif - -#ifndef nssv_CONFIG_USR_SV_OPERATOR -# define nssv_CONFIG_USR_SV_OPERATOR 1 -#endif - -#ifdef nssv_CONFIG_CONVERSION_STD_STRING -# define nssv_CONFIG_CONVERSION_STD_STRING_CLASS_METHODS nssv_CONFIG_CONVERSION_STD_STRING -# define nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS nssv_CONFIG_CONVERSION_STD_STRING -#endif - -#ifndef nssv_CONFIG_CONVERSION_STD_STRING_CLASS_METHODS -# define nssv_CONFIG_CONVERSION_STD_STRING_CLASS_METHODS 1 -#endif - -#ifndef nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS -# define nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS 1 -#endif - -#ifndef nssv_CONFIG_NO_STREAM_INSERTION -# define nssv_CONFIG_NO_STREAM_INSERTION 0 -#endif - -#ifndef nssv_CONFIG_CONSTEXPR11_STD_SEARCH -# define nssv_CONFIG_CONSTEXPR11_STD_SEARCH 1 -#endif - -// Control presence of exception handling (try and auto discover): - -#ifndef nssv_CONFIG_NO_EXCEPTIONS -# if defined(_MSC_VER) -# include // for _HAS_EXCEPTIONS -# endif -# if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || (_HAS_EXCEPTIONS) -# define nssv_CONFIG_NO_EXCEPTIONS 0 -# else -# define nssv_CONFIG_NO_EXCEPTIONS 1 -# endif -#endif - -// C++ language version detection (C++23 is speculative): -// Note: VC14.0/1900 (VS2015) lacks too much from C++14. - -#ifndef nssv_CPLUSPLUS -# if defined(_MSVC_LANG ) && !defined(__clang__) -# define nssv_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG ) -# else -# define nssv_CPLUSPLUS __cplusplus -# endif -#endif - -#define nssv_CPP98_OR_GREATER ( nssv_CPLUSPLUS >= 199711L ) -#define nssv_CPP11_OR_GREATER ( nssv_CPLUSPLUS >= 201103L ) -#define nssv_CPP11_OR_GREATER_ ( nssv_CPLUSPLUS >= 201103L ) -#define nssv_CPP14_OR_GREATER ( nssv_CPLUSPLUS >= 201402L ) -#define nssv_CPP17_OR_GREATER ( nssv_CPLUSPLUS >= 201703L ) -#define nssv_CPP20_OR_GREATER ( nssv_CPLUSPLUS >= 202002L ) -#define nssv_CPP23_OR_GREATER ( nssv_CPLUSPLUS >= 202300L ) - -// use C++17 std::string_view if available and requested: - -#if nssv_CPP17_OR_GREATER && defined(__has_include ) -# if __has_include( ) -# define nssv_HAVE_STD_STRING_VIEW 1 -# else -# define nssv_HAVE_STD_STRING_VIEW 0 -# endif -#else -# define nssv_HAVE_STD_STRING_VIEW 0 -#endif - -#define nssv_USES_STD_STRING_VIEW ( (nssv_CONFIG_SELECT_STRING_VIEW == nssv_STRING_VIEW_STD) || ((nssv_CONFIG_SELECT_STRING_VIEW == nssv_STRING_VIEW_DEFAULT) && nssv_HAVE_STD_STRING_VIEW) ) - -#define nssv_HAVE_STARTS_WITH ( nssv_CPP20_OR_GREATER || !nssv_USES_STD_STRING_VIEW ) -#define nssv_HAVE_ENDS_WITH nssv_HAVE_STARTS_WITH - -// -// Use C++17 std::string_view: -// - -#if nssv_USES_STD_STRING_VIEW - -#include - -// Extensions for std::string: - -#if nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS - -#include - -namespace nonstd { - -template< class CharT, class Traits, class Allocator = std::allocator > -std::basic_string -to_string( std::basic_string_view v, Allocator const & a = Allocator() ) -{ - return std::basic_string( v.begin(), v.end(), a ); -} - -template< class CharT, class Traits, class Allocator > -std::basic_string_view -to_string_view( std::basic_string const & s ) -{ - return std::basic_string_view( s.data(), s.size() ); -} - -// Literal operators sv and _sv: - -#if nssv_CONFIG_STD_SV_OPERATOR - -using namespace std::literals::string_view_literals; - -#endif - -#if nssv_CONFIG_USR_SV_OPERATOR - -inline namespace literals { -inline namespace string_view_literals { - - -constexpr std::string_view operator ""_sv( const char* str, size_t len ) noexcept // (1) -{ - return std::string_view{ str, len }; -} - -constexpr std::u16string_view operator ""_sv( const char16_t* str, size_t len ) noexcept // (2) -{ - return std::u16string_view{ str, len }; -} - -constexpr std::u32string_view operator ""_sv( const char32_t* str, size_t len ) noexcept // (3) -{ - return std::u32string_view{ str, len }; -} - -constexpr std::wstring_view operator ""_sv( const wchar_t* str, size_t len ) noexcept // (4) -{ - return std::wstring_view{ str, len }; -} - -}} // namespace literals::string_view_literals - -#endif // nssv_CONFIG_USR_SV_OPERATOR - -} // namespace nonstd - -#endif // nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS - -namespace nonstd { - -using std::string_view; -using std::wstring_view; -using std::u16string_view; -using std::u32string_view; -using std::basic_string_view; - -// literal "sv" and "_sv", see above - -using std::operator==; -using std::operator!=; -using std::operator<; -using std::operator<=; -using std::operator>; -using std::operator>=; - -using std::operator<<; - -} // namespace nonstd - -#else // nssv_HAVE_STD_STRING_VIEW - -// -// Before C++17: use string_view lite: -// - -// Compiler versions: -// -// MSVC++ 6.0 _MSC_VER == 1200 nssv_COMPILER_MSVC_VERSION == 60 (Visual Studio 6.0) -// MSVC++ 7.0 _MSC_VER == 1300 nssv_COMPILER_MSVC_VERSION == 70 (Visual Studio .NET 2002) -// MSVC++ 7.1 _MSC_VER == 1310 nssv_COMPILER_MSVC_VERSION == 71 (Visual Studio .NET 2003) -// MSVC++ 8.0 _MSC_VER == 1400 nssv_COMPILER_MSVC_VERSION == 80 (Visual Studio 2005) -// MSVC++ 9.0 _MSC_VER == 1500 nssv_COMPILER_MSVC_VERSION == 90 (Visual Studio 2008) -// MSVC++ 10.0 _MSC_VER == 1600 nssv_COMPILER_MSVC_VERSION == 100 (Visual Studio 2010) -// MSVC++ 11.0 _MSC_VER == 1700 nssv_COMPILER_MSVC_VERSION == 110 (Visual Studio 2012) -// MSVC++ 12.0 _MSC_VER == 1800 nssv_COMPILER_MSVC_VERSION == 120 (Visual Studio 2013) -// MSVC++ 14.0 _MSC_VER == 1900 nssv_COMPILER_MSVC_VERSION == 140 (Visual Studio 2015) -// MSVC++ 14.1 _MSC_VER >= 1910 nssv_COMPILER_MSVC_VERSION == 141 (Visual Studio 2017) -// MSVC++ 14.2 _MSC_VER >= 1920 nssv_COMPILER_MSVC_VERSION == 142 (Visual Studio 2019) - -#if defined(_MSC_VER ) && !defined(__clang__) -# define nssv_COMPILER_MSVC_VER (_MSC_VER ) -# define nssv_COMPILER_MSVC_VERSION (_MSC_VER / 10 - 10 * ( 5 + (_MSC_VER < 1900 ) ) ) -#else -# define nssv_COMPILER_MSVC_VER 0 -# define nssv_COMPILER_MSVC_VERSION 0 -#endif - -#define nssv_COMPILER_VERSION( major, minor, patch ) ( 10 * ( 10 * (major) + (minor) ) + (patch) ) - -#if defined( __apple_build_version__ ) -# define nssv_COMPILER_APPLECLANG_VERSION nssv_COMPILER_VERSION(__clang_major__, __clang_minor__, __clang_patchlevel__) -# define nssv_COMPILER_CLANG_VERSION 0 -#elif defined( __clang__ ) -# define nssv_COMPILER_APPLECLANG_VERSION 0 -# define nssv_COMPILER_CLANG_VERSION nssv_COMPILER_VERSION(__clang_major__, __clang_minor__, __clang_patchlevel__) -#else -# define nssv_COMPILER_APPLECLANG_VERSION 0 -# define nssv_COMPILER_CLANG_VERSION 0 -#endif - -#if defined(__GNUC__) && !defined(__clang__) -# define nssv_COMPILER_GNUC_VERSION nssv_COMPILER_VERSION(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) -#else -# define nssv_COMPILER_GNUC_VERSION 0 -#endif - -// half-open range [lo..hi): -#define nssv_BETWEEN( v, lo, hi ) ( (lo) <= (v) && (v) < (hi) ) - -// Presence of language and library features: - -#ifdef _HAS_CPP0X -# define nssv_HAS_CPP0X _HAS_CPP0X -#else -# define nssv_HAS_CPP0X 0 -#endif - -// Unless defined otherwise below, consider VC14 as C++11 for string-view-lite: - -#if nssv_COMPILER_MSVC_VER >= 1900 -# undef nssv_CPP11_OR_GREATER -# define nssv_CPP11_OR_GREATER 1 -#endif - -#define nssv_CPP11_90 (nssv_CPP11_OR_GREATER_ || nssv_COMPILER_MSVC_VER >= 1500) -#define nssv_CPP11_100 (nssv_CPP11_OR_GREATER_ || nssv_COMPILER_MSVC_VER >= 1600) -#define nssv_CPP11_110 (nssv_CPP11_OR_GREATER_ || nssv_COMPILER_MSVC_VER >= 1700) -#define nssv_CPP11_120 (nssv_CPP11_OR_GREATER_ || nssv_COMPILER_MSVC_VER >= 1800) -#define nssv_CPP11_140 (nssv_CPP11_OR_GREATER_ || nssv_COMPILER_MSVC_VER >= 1900) -#define nssv_CPP11_141 (nssv_CPP11_OR_GREATER_ || nssv_COMPILER_MSVC_VER >= 1910) - -#define nssv_CPP14_000 (nssv_CPP14_OR_GREATER) -#define nssv_CPP17_000 (nssv_CPP17_OR_GREATER) - -// Presence of C++11 language features: - -#define nssv_HAVE_CONSTEXPR_11 nssv_CPP11_140 -#define nssv_HAVE_EXPLICIT_CONVERSION nssv_CPP11_140 -#define nssv_HAVE_INLINE_NAMESPACE nssv_CPP11_140 -#define nssv_HAVE_IS_DEFAULT nssv_CPP11_140 -#define nssv_HAVE_IS_DELETE nssv_CPP11_140 -#define nssv_HAVE_NOEXCEPT nssv_CPP11_140 -#define nssv_HAVE_NULLPTR nssv_CPP11_100 -#define nssv_HAVE_REF_QUALIFIER nssv_CPP11_140 -#define nssv_HAVE_UNICODE_LITERALS nssv_CPP11_140 -#define nssv_HAVE_USER_DEFINED_LITERALS nssv_CPP11_140 -#define nssv_HAVE_WCHAR16_T nssv_CPP11_100 -#define nssv_HAVE_WCHAR32_T nssv_CPP11_100 - -#if ! ( ( nssv_CPP11_OR_GREATER && nssv_COMPILER_CLANG_VERSION ) || nssv_BETWEEN( nssv_COMPILER_CLANG_VERSION, 300, 400 ) ) -# define nssv_HAVE_STD_DEFINED_LITERALS nssv_CPP11_140 -#else -# define nssv_HAVE_STD_DEFINED_LITERALS 0 -#endif - -// Presence of C++14 language features: - -#define nssv_HAVE_CONSTEXPR_14 nssv_CPP14_000 - -// Presence of C++17 language features: - -#define nssv_HAVE_NODISCARD nssv_CPP17_000 - -// Presence of C++ library features: - -#define nssv_HAVE_STD_HASH nssv_CPP11_120 - -// Presence of compiler intrinsics: - -// Providing char-type specializations for compare() and length() that -// use compiler intrinsics can improve compile- and run-time performance. -// -// The challenge is in using the right combinations of builtin availability -// and its constexpr-ness. -// -// | compiler | __builtin_memcmp (constexpr) | memcmp (constexpr) | -// |----------|------------------------------|---------------------| -// | clang | 4.0 (>= 4.0 ) | any (? ) | -// | clang-a | 9.0 (>= 9.0 ) | any (? ) | -// | gcc | any (constexpr) | any (? ) | -// | msvc | >= 14.2 C++17 (>= 14.2 ) | any (? ) | - -#define nssv_HAVE_BUILTIN_VER ( (nssv_CPP17_000 && nssv_COMPILER_MSVC_VERSION >= 142) || nssv_COMPILER_GNUC_VERSION > 0 || nssv_COMPILER_CLANG_VERSION >= 400 || nssv_COMPILER_APPLECLANG_VERSION >= 900 ) -#define nssv_HAVE_BUILTIN_CE ( nssv_HAVE_BUILTIN_VER ) - -#define nssv_HAVE_BUILTIN_MEMCMP ( (nssv_HAVE_CONSTEXPR_14 && nssv_HAVE_BUILTIN_CE) || !nssv_HAVE_CONSTEXPR_14 ) -#define nssv_HAVE_BUILTIN_STRLEN ( (nssv_HAVE_CONSTEXPR_11 && nssv_HAVE_BUILTIN_CE) || !nssv_HAVE_CONSTEXPR_11 ) - -#ifdef __has_builtin -# define nssv_HAVE_BUILTIN( x ) __has_builtin( x ) -#else -# define nssv_HAVE_BUILTIN( x ) 0 -#endif - -#if nssv_HAVE_BUILTIN(__builtin_memcmp) || nssv_HAVE_BUILTIN_VER -# define nssv_BUILTIN_MEMCMP __builtin_memcmp -#else -# define nssv_BUILTIN_MEMCMP memcmp -#endif - -#if nssv_HAVE_BUILTIN(__builtin_strlen) || nssv_HAVE_BUILTIN_VER -# define nssv_BUILTIN_STRLEN __builtin_strlen -#else -# define nssv_BUILTIN_STRLEN strlen -#endif - -// C++ feature usage: - -#if nssv_HAVE_CONSTEXPR_11 -# define nssv_constexpr constexpr -#else -# define nssv_constexpr /*constexpr*/ -#endif - -#if nssv_HAVE_CONSTEXPR_14 -# define nssv_constexpr14 constexpr -#else -# define nssv_constexpr14 /*constexpr*/ -#endif - -#if nssv_HAVE_EXPLICIT_CONVERSION -# define nssv_explicit explicit -#else -# define nssv_explicit /*explicit*/ -#endif - -#if nssv_HAVE_INLINE_NAMESPACE -# define nssv_inline_ns inline -#else -# define nssv_inline_ns /*inline*/ -#endif - -#if nssv_HAVE_NOEXCEPT -# define nssv_noexcept noexcept -#else -# define nssv_noexcept /*noexcept*/ -#endif - -//#if nssv_HAVE_REF_QUALIFIER -//# define nssv_ref_qual & -//# define nssv_refref_qual && -//#else -//# define nssv_ref_qual /*&*/ -//# define nssv_refref_qual /*&&*/ -//#endif - -#if nssv_HAVE_NULLPTR -# define nssv_nullptr nullptr -#else -# define nssv_nullptr NULL -#endif - -#if nssv_HAVE_NODISCARD -# define nssv_nodiscard simdjson_warn_unused -#else -# define nssv_nodiscard /*simdjson_warn_unused*/ -#endif - -// Additional includes: - -#include -#include -#include -#include -#include // std::char_traits<> - -#if ! nssv_CONFIG_NO_STREAM_INSERTION -# include -#endif - -#if ! nssv_CONFIG_NO_EXCEPTIONS -# include -#endif - -#if nssv_CPP11_OR_GREATER -# include -#endif - -// Clang, GNUC, MSVC warning suppression macros: - -#if defined(__clang__) -# pragma clang diagnostic ignored "-Wreserved-user-defined-literal" -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wuser-defined-literals" -#elif nssv_COMPILER_GNUC_VERSION >= 480 -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wliteral-suffix" -#endif // __clang__ - -#if nssv_COMPILER_MSVC_VERSION >= 140 -# define nssv_SUPPRESS_MSGSL_WARNING(expr) [[gsl::suppress(expr)]] -# define nssv_SUPPRESS_MSVC_WARNING(code, descr) __pragma(warning(suppress: code) ) -# define nssv_DISABLE_MSVC_WARNINGS(codes) __pragma(warning(push)) __pragma(warning(disable: codes)) -#else -# define nssv_SUPPRESS_MSGSL_WARNING(expr) -# define nssv_SUPPRESS_MSVC_WARNING(code, descr) -# define nssv_DISABLE_MSVC_WARNINGS(codes) -#endif - -#if defined(__clang__) -# define nssv_RESTORE_WARNINGS() _Pragma("clang diagnostic pop") -#elif nssv_COMPILER_GNUC_VERSION >= 480 -# define nssv_RESTORE_WARNINGS() _Pragma("GCC diagnostic pop") -#elif nssv_COMPILER_MSVC_VERSION >= 140 -# define nssv_RESTORE_WARNINGS() __pragma(warning(pop )) -#else -# define nssv_RESTORE_WARNINGS() -#endif - -// Suppress the following MSVC (GSL) warnings: -// - C4455, non-gsl : 'operator ""sv': literal suffix identifiers that do not -// start with an underscore are reserved -// - C26472, gsl::t.1 : don't use a static_cast for arithmetic conversions; -// use brace initialization, gsl::narrow_cast or gsl::narow -// - C26481: gsl::b.1 : don't use pointer arithmetic. Use span instead - -nssv_DISABLE_MSVC_WARNINGS( 4455 26481 26472 ) -//nssv_DISABLE_CLANG_WARNINGS( "-Wuser-defined-literals" ) -//nssv_DISABLE_GNUC_WARNINGS( -Wliteral-suffix ) - -namespace nonstd { namespace sv_lite { - -// -// basic_string_view declaration: -// - -template -< - class CharT, - class Traits = std::char_traits -> -class basic_string_view; - -namespace detail { - -// support constexpr comparison in C++14; -// for C++17 and later, use provided traits: - -template< typename CharT > -inline nssv_constexpr14 int compare( CharT const * s1, CharT const * s2, std::size_t count ) -{ - while ( count-- != 0 ) - { - if ( *s1 < *s2 ) return -1; - if ( *s1 > *s2 ) return +1; - ++s1; ++s2; - } - return 0; -} - -#if nssv_HAVE_BUILTIN_MEMCMP - -// specialization of compare() for char, see also generic compare() above: - -inline nssv_constexpr14 int compare( char const * s1, char const * s2, std::size_t count ) -{ - return nssv_BUILTIN_MEMCMP( s1, s2, count ); -} - -#endif - -#if nssv_HAVE_BUILTIN_STRLEN - -// specialization of length() for char, see also generic length() further below: - -inline nssv_constexpr std::size_t length( char const * s ) -{ - return nssv_BUILTIN_STRLEN( s ); -} - -#endif - -#if defined(__OPTIMIZE__) - -// gcc, clang provide __OPTIMIZE__ -// Expect tail call optimization to make length() non-recursive: - -template< typename CharT > -inline nssv_constexpr std::size_t length( CharT * s, std::size_t result = 0 ) -{ - return *s == '\0' ? result : length( s + 1, result + 1 ); -} - -#else // OPTIMIZE - -// non-recursive: - -template< typename CharT > -inline nssv_constexpr14 std::size_t length( CharT * s ) -{ - std::size_t result = 0; - while ( *s++ != '\0' ) - { - ++result; - } - return result; -} - -#endif // OPTIMIZE - -#if nssv_CPP11_OR_GREATER && ! nssv_CPP17_OR_GREATER -#if defined(__OPTIMIZE__) - -// gcc, clang provide __OPTIMIZE__ -// Expect tail call optimization to make search() non-recursive: - -template< class CharT, class Traits = std::char_traits > -constexpr const CharT* search( basic_string_view haystack, basic_string_view needle ) -{ - return haystack.starts_with( needle ) ? haystack.begin() : - haystack.empty() ? haystack.end() : search( haystack.substr(1), needle ); -} - -#else // OPTIMIZE - -// non-recursive: - -#if nssv_CONFIG_CONSTEXPR11_STD_SEARCH - -template< class CharT, class Traits = std::char_traits > -constexpr const CharT* search( basic_string_view haystack, basic_string_view needle ) -{ - return std::search( haystack.begin(), haystack.end(), needle.begin(), needle.end() ); -} - -#else // nssv_CONFIG_CONSTEXPR11_STD_SEARCH - -template< class CharT, class Traits = std::char_traits > -nssv_constexpr14 const CharT* search( basic_string_view haystack, basic_string_view needle ) -{ - while ( needle.size() <= haystack.size() ) - { - if ( haystack.starts_with(needle) ) - { - return haystack.cbegin(); - } - haystack = basic_string_view{ haystack.begin() + 1, haystack.size() - 1U }; - } - return haystack.cend(); -} -#endif // nssv_CONFIG_CONSTEXPR11_STD_SEARCH - -#endif // OPTIMIZE -#endif // nssv_CPP11_OR_GREATER && ! nssv_CPP17_OR_GREATER - -} // namespace detail - -// -// basic_string_view: -// - -template -< - class CharT, - class Traits /* = std::char_traits */ -> -class basic_string_view -{ -public: - // Member types: - - typedef Traits traits_type; - typedef CharT value_type; - - typedef CharT * pointer; - typedef CharT const * const_pointer; - typedef CharT & reference; - typedef CharT const & const_reference; - - typedef const_pointer iterator; - typedef const_pointer const_iterator; - typedef std::reverse_iterator< const_iterator > reverse_iterator; - typedef std::reverse_iterator< const_iterator > const_reverse_iterator; - - typedef std::size_t size_type; - typedef std::ptrdiff_t difference_type; - - // 24.4.2.1 Construction and assignment: - - nssv_constexpr basic_string_view() nssv_noexcept - : data_( nssv_nullptr ) - , size_( 0 ) - {} - -#if nssv_CPP11_OR_GREATER - nssv_constexpr basic_string_view( basic_string_view const & other ) nssv_noexcept = default; -#else - nssv_constexpr basic_string_view( basic_string_view const & other ) nssv_noexcept - : data_( other.data_) - , size_( other.size_) - {} -#endif - - nssv_constexpr basic_string_view( CharT const * s, size_type count ) nssv_noexcept // non-standard noexcept - : data_( s ) - , size_( count ) - {} - - nssv_constexpr basic_string_view( CharT const * s) nssv_noexcept // non-standard noexcept - : data_( s ) -#if nssv_CPP17_OR_GREATER - , size_( Traits::length(s) ) -#elif nssv_CPP11_OR_GREATER - , size_( detail::length(s) ) -#else - , size_( Traits::length(s) ) -#endif - {} - -#if nssv_HAVE_NULLPTR -# if nssv_HAVE_IS_DELETE - nssv_constexpr basic_string_view( std::nullptr_t ) nssv_noexcept = delete; -# else - private: nssv_constexpr basic_string_view( std::nullptr_t ) nssv_noexcept; public: -# endif -#endif - - // Assignment: - -#if nssv_CPP11_OR_GREATER - nssv_constexpr14 basic_string_view & operator=( basic_string_view const & other ) nssv_noexcept = default; -#else - nssv_constexpr14 basic_string_view & operator=( basic_string_view const & other ) nssv_noexcept - { - data_ = other.data_; - size_ = other.size_; - return *this; - } -#endif - - // 24.4.2.2 Iterator support: - - nssv_constexpr const_iterator begin() const nssv_noexcept { return data_; } - nssv_constexpr const_iterator end() const nssv_noexcept { return data_ + size_; } - - nssv_constexpr const_iterator cbegin() const nssv_noexcept { return begin(); } - nssv_constexpr const_iterator cend() const nssv_noexcept { return end(); } - - nssv_constexpr const_reverse_iterator rbegin() const nssv_noexcept { return const_reverse_iterator( end() ); } - nssv_constexpr const_reverse_iterator rend() const nssv_noexcept { return const_reverse_iterator( begin() ); } - - nssv_constexpr const_reverse_iterator crbegin() const nssv_noexcept { return rbegin(); } - nssv_constexpr const_reverse_iterator crend() const nssv_noexcept { return rend(); } - - // 24.4.2.3 Capacity: - - nssv_constexpr size_type size() const nssv_noexcept { return size_; } - nssv_constexpr size_type length() const nssv_noexcept { return size_; } - nssv_constexpr size_type max_size() const nssv_noexcept { return (std::numeric_limits< size_type >::max)(); } - - // since C++20 - nssv_nodiscard nssv_constexpr bool empty() const nssv_noexcept - { - return 0 == size_; - } - - // 24.4.2.4 Element access: - - nssv_constexpr const_reference operator[]( size_type pos ) const - { - return data_at( pos ); - } - - nssv_constexpr14 const_reference at( size_type pos ) const - { -#if nssv_CONFIG_NO_EXCEPTIONS - assert( pos < size() ); -#else - if ( pos >= size() ) - { - throw std::out_of_range("nonstd::string_view::at()"); - } -#endif - return data_at( pos ); - } - - nssv_constexpr const_reference front() const { return data_at( 0 ); } - nssv_constexpr const_reference back() const { return data_at( size() - 1 ); } - - nssv_constexpr const_pointer data() const nssv_noexcept { return data_; } - - // 24.4.2.5 Modifiers: - - nssv_constexpr14 void remove_prefix( size_type n ) - { - assert( n <= size() ); - data_ += n; - size_ -= n; - } - - nssv_constexpr14 void remove_suffix( size_type n ) - { - assert( n <= size() ); - size_ -= n; - } - - nssv_constexpr14 void swap( basic_string_view & other ) nssv_noexcept - { - const basic_string_view tmp(other); - other = *this; - *this = tmp; - } - - // 24.4.2.6 String operations: - - size_type copy( CharT * dest, size_type n, size_type pos = 0 ) const - { -#if nssv_CONFIG_NO_EXCEPTIONS - assert( pos <= size() ); -#else - if ( pos > size() ) - { - throw std::out_of_range("nonstd::string_view::copy()"); - } -#endif - const size_type rlen = (std::min)( n, size() - pos ); - - (void) Traits::copy( dest, data() + pos, rlen ); - - return rlen; - } - - nssv_constexpr14 basic_string_view substr( size_type pos = 0, size_type n = npos ) const - { -#if nssv_CONFIG_NO_EXCEPTIONS - assert( pos <= size() ); -#else - if ( pos > size() ) - { - throw std::out_of_range("nonstd::string_view::substr()"); - } -#endif - return basic_string_view( data() + pos, (std::min)( n, size() - pos ) ); - } - - // compare(), 6x: - - nssv_constexpr14 int compare( basic_string_view other ) const nssv_noexcept // (1) - { -#if nssv_CPP17_OR_GREATER - if ( const int result = Traits::compare( data(), other.data(), (std::min)( size(), other.size() ) ) ) -#else - if ( const int result = detail::compare( data(), other.data(), (std::min)( size(), other.size() ) ) ) -#endif - { - return result; - } - - return size() == other.size() ? 0 : size() < other.size() ? -1 : 1; - } - - nssv_constexpr int compare( size_type pos1, size_type n1, basic_string_view other ) const // (2) - { - return substr( pos1, n1 ).compare( other ); - } - - nssv_constexpr int compare( size_type pos1, size_type n1, basic_string_view other, size_type pos2, size_type n2 ) const // (3) - { - return substr( pos1, n1 ).compare( other.substr( pos2, n2 ) ); - } - - nssv_constexpr int compare( CharT const * s ) const // (4) - { - return compare( basic_string_view( s ) ); - } - - nssv_constexpr int compare( size_type pos1, size_type n1, CharT const * s ) const // (5) - { - return substr( pos1, n1 ).compare( basic_string_view( s ) ); - } - - nssv_constexpr int compare( size_type pos1, size_type n1, CharT const * s, size_type n2 ) const // (6) - { - return substr( pos1, n1 ).compare( basic_string_view( s, n2 ) ); - } - - // 24.4.2.7 Searching: - - // starts_with(), 3x, since C++20: - - nssv_constexpr bool starts_with( basic_string_view v ) const nssv_noexcept // (1) - { - return size() >= v.size() && compare( 0, v.size(), v ) == 0; - } - - nssv_constexpr bool starts_with( CharT c ) const nssv_noexcept // (2) - { - return starts_with( basic_string_view( &c, 1 ) ); - } - - nssv_constexpr bool starts_with( CharT const * s ) const // (3) - { - return starts_with( basic_string_view( s ) ); - } - - // ends_with(), 3x, since C++20: - - nssv_constexpr bool ends_with( basic_string_view v ) const nssv_noexcept // (1) - { - return size() >= v.size() && compare( size() - v.size(), npos, v ) == 0; - } - - nssv_constexpr bool ends_with( CharT c ) const nssv_noexcept // (2) - { - return ends_with( basic_string_view( &c, 1 ) ); - } - - nssv_constexpr bool ends_with( CharT const * s ) const // (3) - { - return ends_with( basic_string_view( s ) ); - } - - // find(), 4x: - - nssv_constexpr14 size_type find( basic_string_view v, size_type pos = 0 ) const nssv_noexcept // (1) - { - return assert( v.size() == 0 || v.data() != nssv_nullptr ) - , pos >= size() - ? npos : to_pos( -#if nssv_CPP11_OR_GREATER && ! nssv_CPP17_OR_GREATER - detail::search( substr(pos), v ) -#else - std::search( cbegin() + pos, cend(), v.cbegin(), v.cend(), Traits::eq ) -#endif - ); - } - - nssv_constexpr size_type find( CharT c, size_type pos = 0 ) const nssv_noexcept // (2) - { - return find( basic_string_view( &c, 1 ), pos ); - } - - nssv_constexpr size_type find( CharT const * s, size_type pos, size_type n ) const // (3) - { - return find( basic_string_view( s, n ), pos ); - } - - nssv_constexpr size_type find( CharT const * s, size_type pos = 0 ) const // (4) - { - return find( basic_string_view( s ), pos ); - } - - // rfind(), 4x: - - nssv_constexpr14 size_type rfind( basic_string_view v, size_type pos = npos ) const nssv_noexcept // (1) - { - if ( size() < v.size() ) - { - return npos; - } - - if ( v.empty() ) - { - return (std::min)( size(), pos ); - } - - const_iterator last = cbegin() + (std::min)( size() - v.size(), pos ) + v.size(); - const_iterator result = std::find_end( cbegin(), last, v.cbegin(), v.cend(), Traits::eq ); - - return result != last ? size_type( result - cbegin() ) : npos; - } - - nssv_constexpr14 size_type rfind( CharT c, size_type pos = npos ) const nssv_noexcept // (2) - { - return rfind( basic_string_view( &c, 1 ), pos ); - } - - nssv_constexpr14 size_type rfind( CharT const * s, size_type pos, size_type n ) const // (3) - { - return rfind( basic_string_view( s, n ), pos ); - } - - nssv_constexpr14 size_type rfind( CharT const * s, size_type pos = npos ) const // (4) - { - return rfind( basic_string_view( s ), pos ); - } - - // find_first_of(), 4x: - - nssv_constexpr size_type find_first_of( basic_string_view v, size_type pos = 0 ) const nssv_noexcept // (1) - { - return pos >= size() - ? npos - : to_pos( std::find_first_of( cbegin() + pos, cend(), v.cbegin(), v.cend(), Traits::eq ) ); - } - - nssv_constexpr size_type find_first_of( CharT c, size_type pos = 0 ) const nssv_noexcept // (2) - { - return find_first_of( basic_string_view( &c, 1 ), pos ); - } - - nssv_constexpr size_type find_first_of( CharT const * s, size_type pos, size_type n ) const // (3) - { - return find_first_of( basic_string_view( s, n ), pos ); - } - - nssv_constexpr size_type find_first_of( CharT const * s, size_type pos = 0 ) const // (4) - { - return find_first_of( basic_string_view( s ), pos ); - } - - // find_last_of(), 4x: - - nssv_constexpr size_type find_last_of( basic_string_view v, size_type pos = npos ) const nssv_noexcept // (1) - { - return empty() - ? npos - : pos >= size() - ? find_last_of( v, size() - 1 ) - : to_pos( std::find_first_of( const_reverse_iterator( cbegin() + pos + 1 ), crend(), v.cbegin(), v.cend(), Traits::eq ) ); - } - - nssv_constexpr size_type find_last_of( CharT c, size_type pos = npos ) const nssv_noexcept // (2) - { - return find_last_of( basic_string_view( &c, 1 ), pos ); - } - - nssv_constexpr size_type find_last_of( CharT const * s, size_type pos, size_type count ) const // (3) - { - return find_last_of( basic_string_view( s, count ), pos ); - } - - nssv_constexpr size_type find_last_of( CharT const * s, size_type pos = npos ) const // (4) - { - return find_last_of( basic_string_view( s ), pos ); - } - - // find_first_not_of(), 4x: - - nssv_constexpr size_type find_first_not_of( basic_string_view v, size_type pos = 0 ) const nssv_noexcept // (1) - { - return pos >= size() - ? npos - : to_pos( std::find_if( cbegin() + pos, cend(), not_in_view( v ) ) ); - } - - nssv_constexpr size_type find_first_not_of( CharT c, size_type pos = 0 ) const nssv_noexcept // (2) - { - return find_first_not_of( basic_string_view( &c, 1 ), pos ); - } - - nssv_constexpr size_type find_first_not_of( CharT const * s, size_type pos, size_type count ) const // (3) - { - return find_first_not_of( basic_string_view( s, count ), pos ); - } - - nssv_constexpr size_type find_first_not_of( CharT const * s, size_type pos = 0 ) const // (4) - { - return find_first_not_of( basic_string_view( s ), pos ); - } - - // find_last_not_of(), 4x: - - nssv_constexpr size_type find_last_not_of( basic_string_view v, size_type pos = npos ) const nssv_noexcept // (1) - { - return empty() - ? npos - : pos >= size() - ? find_last_not_of( v, size() - 1 ) - : to_pos( std::find_if( const_reverse_iterator( cbegin() + pos + 1 ), crend(), not_in_view( v ) ) ); - } - - nssv_constexpr size_type find_last_not_of( CharT c, size_type pos = npos ) const nssv_noexcept // (2) - { - return find_last_not_of( basic_string_view( &c, 1 ), pos ); - } - - nssv_constexpr size_type find_last_not_of( CharT const * s, size_type pos, size_type count ) const // (3) - { - return find_last_not_of( basic_string_view( s, count ), pos ); - } - - nssv_constexpr size_type find_last_not_of( CharT const * s, size_type pos = npos ) const // (4) - { - return find_last_not_of( basic_string_view( s ), pos ); - } - - // Constants: - -#if nssv_CPP17_OR_GREATER - static nssv_constexpr size_type npos = size_type(-1); -#elif nssv_CPP11_OR_GREATER - enum : size_type { npos = size_type(-1) }; -#else - enum { npos = size_type(-1) }; -#endif - -private: - struct not_in_view - { - const basic_string_view v; - - nssv_constexpr explicit not_in_view( basic_string_view v_ ) : v( v_ ) {} - - nssv_constexpr bool operator()( CharT c ) const - { - return npos == v.find_first_of( c ); - } - }; - - nssv_constexpr size_type to_pos( const_iterator it ) const - { - return it == cend() ? npos : size_type( it - cbegin() ); - } - - nssv_constexpr size_type to_pos( const_reverse_iterator it ) const - { - return it == crend() ? npos : size_type( crend() - it - 1 ); - } - - nssv_constexpr const_reference data_at( size_type pos ) const - { -#if nssv_BETWEEN( nssv_COMPILER_GNUC_VERSION, 1, 500 ) - return data_[pos]; -#else - return assert( pos < size() ), data_[pos]; -#endif - } - -private: - const_pointer data_; - size_type size_; - -public: -#if nssv_CONFIG_CONVERSION_STD_STRING_CLASS_METHODS - - template< class Allocator > - basic_string_view( std::basic_string const & s ) nssv_noexcept - : data_( s.data() ) - , size_( s.size() ) - {} - -#if nssv_HAVE_EXPLICIT_CONVERSION - - template< class Allocator > - explicit operator std::basic_string() const - { - return to_string( Allocator() ); - } - -#endif // nssv_HAVE_EXPLICIT_CONVERSION - -#if nssv_CPP11_OR_GREATER - - template< class Allocator = std::allocator > - std::basic_string - to_string( Allocator const & a = Allocator() ) const - { - return std::basic_string( begin(), end(), a ); - } - -#else - - std::basic_string - to_string() const - { - return std::basic_string( begin(), end() ); - } - - template< class Allocator > - std::basic_string - to_string( Allocator const & a ) const - { - return std::basic_string( begin(), end(), a ); - } - -#endif // nssv_CPP11_OR_GREATER - -#endif // nssv_CONFIG_CONVERSION_STD_STRING_CLASS_METHODS -}; - -// -// Non-member functions: -// - -// 24.4.3 Non-member comparison functions: -// lexicographically compare two string views (function template): - -template< class CharT, class Traits > -nssv_constexpr bool operator== ( - basic_string_view lhs, - basic_string_view rhs ) nssv_noexcept -{ return lhs.size() == rhs.size() && lhs.compare( rhs ) == 0; } - -template< class CharT, class Traits > -nssv_constexpr bool operator!= ( - basic_string_view lhs, - basic_string_view rhs ) nssv_noexcept -{ return !( lhs == rhs ); } - -template< class CharT, class Traits > -nssv_constexpr bool operator< ( - basic_string_view lhs, - basic_string_view rhs ) nssv_noexcept -{ return lhs.compare( rhs ) < 0; } - -template< class CharT, class Traits > -nssv_constexpr bool operator<= ( - basic_string_view lhs, - basic_string_view rhs ) nssv_noexcept -{ return lhs.compare( rhs ) <= 0; } - -template< class CharT, class Traits > -nssv_constexpr bool operator> ( - basic_string_view lhs, - basic_string_view rhs ) nssv_noexcept -{ return lhs.compare( rhs ) > 0; } - -template< class CharT, class Traits > -nssv_constexpr bool operator>= ( - basic_string_view lhs, - basic_string_view rhs ) nssv_noexcept -{ return lhs.compare( rhs ) >= 0; } - -// Let S be basic_string_view, and sv be an instance of S. -// Implementations shall provide sufficient additional overloads marked -// constexpr and noexcept so that an object t with an implicit conversion -// to S can be compared according to Table 67. - -#if ! nssv_CPP11_OR_GREATER || nssv_BETWEEN( nssv_COMPILER_MSVC_VERSION, 100, 141 ) - -// accommodate for older compilers: - -// == - -template< class CharT, class Traits> -nssv_constexpr bool operator==( - basic_string_view lhs, - CharT const * rhs ) nssv_noexcept -{ return lhs.size() == detail::length( rhs ) && lhs.compare( rhs ) == 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator==( - CharT const * lhs, - basic_string_view rhs ) nssv_noexcept -{ return detail::length( lhs ) == rhs.size() && rhs.compare( lhs ) == 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator==( - basic_string_view lhs, - std::basic_string rhs ) nssv_noexcept -{ return lhs.size() == rhs.size() && lhs.compare( rhs ) == 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator==( - std::basic_string rhs, - basic_string_view lhs ) nssv_noexcept -{ return lhs.size() == rhs.size() && lhs.compare( rhs ) == 0; } - -// != - -template< class CharT, class Traits> -nssv_constexpr bool operator!=( - basic_string_view lhs, - CharT const * rhs ) nssv_noexcept -{ return !( lhs == rhs ); } - -template< class CharT, class Traits> -nssv_constexpr bool operator!=( - CharT const * lhs, - basic_string_view rhs ) nssv_noexcept -{ return !( lhs == rhs ); } - -template< class CharT, class Traits> -nssv_constexpr bool operator!=( - basic_string_view lhs, - std::basic_string rhs ) nssv_noexcept -{ return !( lhs == rhs ); } - -template< class CharT, class Traits> -nssv_constexpr bool operator!=( - std::basic_string rhs, - basic_string_view lhs ) nssv_noexcept -{ return !( lhs == rhs ); } - -// < - -template< class CharT, class Traits> -nssv_constexpr bool operator<( - basic_string_view lhs, - CharT const * rhs ) nssv_noexcept -{ return lhs.compare( rhs ) < 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator<( - CharT const * lhs, - basic_string_view rhs ) nssv_noexcept -{ return rhs.compare( lhs ) > 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator<( - basic_string_view lhs, - std::basic_string rhs ) nssv_noexcept -{ return lhs.compare( rhs ) < 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator<( - std::basic_string rhs, - basic_string_view lhs ) nssv_noexcept -{ return rhs.compare( lhs ) > 0; } - -// <= - -template< class CharT, class Traits> -nssv_constexpr bool operator<=( - basic_string_view lhs, - CharT const * rhs ) nssv_noexcept -{ return lhs.compare( rhs ) <= 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator<=( - CharT const * lhs, - basic_string_view rhs ) nssv_noexcept -{ return rhs.compare( lhs ) >= 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator<=( - basic_string_view lhs, - std::basic_string rhs ) nssv_noexcept -{ return lhs.compare( rhs ) <= 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator<=( - std::basic_string rhs, - basic_string_view lhs ) nssv_noexcept -{ return rhs.compare( lhs ) >= 0; } - -// > - -template< class CharT, class Traits> -nssv_constexpr bool operator>( - basic_string_view lhs, - CharT const * rhs ) nssv_noexcept -{ return lhs.compare( rhs ) > 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator>( - CharT const * lhs, - basic_string_view rhs ) nssv_noexcept -{ return rhs.compare( lhs ) < 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator>( - basic_string_view lhs, - std::basic_string rhs ) nssv_noexcept -{ return lhs.compare( rhs ) > 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator>( - std::basic_string rhs, - basic_string_view lhs ) nssv_noexcept -{ return rhs.compare( lhs ) < 0; } - -// >= - -template< class CharT, class Traits> -nssv_constexpr bool operator>=( - basic_string_view lhs, - CharT const * rhs ) nssv_noexcept -{ return lhs.compare( rhs ) >= 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator>=( - CharT const * lhs, - basic_string_view rhs ) nssv_noexcept -{ return rhs.compare( lhs ) <= 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator>=( - basic_string_view lhs, - std::basic_string rhs ) nssv_noexcept -{ return lhs.compare( rhs ) >= 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator>=( - std::basic_string rhs, - basic_string_view lhs ) nssv_noexcept -{ return rhs.compare( lhs ) <= 0; } - -#else // newer compilers: - -#define nssv_BASIC_STRING_VIEW_I(T,U) typename std::decay< basic_string_view >::type - -#if defined(_MSC_VER) // issue 40 -# define nssv_MSVC_ORDER(x) , int=x -#else -# define nssv_MSVC_ORDER(x) /*, int=x*/ -#endif - -// == - -template< class CharT, class Traits nssv_MSVC_ORDER(1) > -nssv_constexpr bool operator==( - basic_string_view lhs, - nssv_BASIC_STRING_VIEW_I(CharT, Traits) rhs ) nssv_noexcept -{ return lhs.size() == rhs.size() && lhs.compare( rhs ) == 0; } - -template< class CharT, class Traits nssv_MSVC_ORDER(2) > -nssv_constexpr bool operator==( - nssv_BASIC_STRING_VIEW_I(CharT, Traits) lhs, - basic_string_view rhs ) nssv_noexcept -{ return lhs.size() == rhs.size() && lhs.compare( rhs ) == 0; } - -// != - -template< class CharT, class Traits nssv_MSVC_ORDER(1) > -nssv_constexpr bool operator!= ( - basic_string_view < CharT, Traits > lhs, - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) rhs ) nssv_noexcept -{ return !( lhs == rhs ); } - -template< class CharT, class Traits nssv_MSVC_ORDER(2) > -nssv_constexpr bool operator!= ( - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) lhs, - basic_string_view < CharT, Traits > rhs ) nssv_noexcept -{ return !( lhs == rhs ); } - -// < - -template< class CharT, class Traits nssv_MSVC_ORDER(1) > -nssv_constexpr bool operator< ( - basic_string_view < CharT, Traits > lhs, - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) rhs ) nssv_noexcept -{ return lhs.compare( rhs ) < 0; } - -template< class CharT, class Traits nssv_MSVC_ORDER(2) > -nssv_constexpr bool operator< ( - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) lhs, - basic_string_view < CharT, Traits > rhs ) nssv_noexcept -{ return lhs.compare( rhs ) < 0; } - -// <= - -template< class CharT, class Traits nssv_MSVC_ORDER(1) > -nssv_constexpr bool operator<= ( - basic_string_view < CharT, Traits > lhs, - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) rhs ) nssv_noexcept -{ return lhs.compare( rhs ) <= 0; } - -template< class CharT, class Traits nssv_MSVC_ORDER(2) > -nssv_constexpr bool operator<= ( - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) lhs, - basic_string_view < CharT, Traits > rhs ) nssv_noexcept -{ return lhs.compare( rhs ) <= 0; } - -// > - -template< class CharT, class Traits nssv_MSVC_ORDER(1) > -nssv_constexpr bool operator> ( - basic_string_view < CharT, Traits > lhs, - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) rhs ) nssv_noexcept -{ return lhs.compare( rhs ) > 0; } - -template< class CharT, class Traits nssv_MSVC_ORDER(2) > -nssv_constexpr bool operator> ( - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) lhs, - basic_string_view < CharT, Traits > rhs ) nssv_noexcept -{ return lhs.compare( rhs ) > 0; } - -// >= - -template< class CharT, class Traits nssv_MSVC_ORDER(1) > -nssv_constexpr bool operator>= ( - basic_string_view < CharT, Traits > lhs, - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) rhs ) nssv_noexcept -{ return lhs.compare( rhs ) >= 0; } - -template< class CharT, class Traits nssv_MSVC_ORDER(2) > -nssv_constexpr bool operator>= ( - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) lhs, - basic_string_view < CharT, Traits > rhs ) nssv_noexcept -{ return lhs.compare( rhs ) >= 0; } - -#undef nssv_MSVC_ORDER -#undef nssv_BASIC_STRING_VIEW_I - -#endif // compiler-dependent approach to comparisons - -// 24.4.4 Inserters and extractors: - -#if ! nssv_CONFIG_NO_STREAM_INSERTION - -namespace detail { - -template< class Stream > -void write_padding( Stream & os, std::streamsize n ) -{ - for ( std::streamsize i = 0; i < n; ++i ) - os.rdbuf()->sputc( os.fill() ); -} - -template< class Stream, class View > -Stream & write_to_stream( Stream & os, View const & sv ) -{ - typename Stream::sentry sentry( os ); - - if ( !sentry ) - return os; - - const std::streamsize length = static_cast( sv.length() ); - - // Whether, and how, to pad: - const bool pad = ( length < os.width() ); - const bool left_pad = pad && ( os.flags() & std::ios_base::adjustfield ) == std::ios_base::right; - - if ( left_pad ) - write_padding( os, os.width() - length ); - - // Write span characters: - os.rdbuf()->sputn( sv.begin(), length ); - - if ( pad && !left_pad ) - write_padding( os, os.width() - length ); - - // Reset output stream width: - os.width( 0 ); - - return os; -} - -} // namespace detail - -template< class CharT, class Traits > -std::basic_ostream & -operator<<( - std::basic_ostream& os, - basic_string_view sv ) -{ - return detail::write_to_stream( os, sv ); -} - -#endif // nssv_CONFIG_NO_STREAM_INSERTION - -// Several typedefs for common character types are provided: - -typedef basic_string_view string_view; -typedef basic_string_view wstring_view; -#if nssv_HAVE_WCHAR16_T -typedef basic_string_view u16string_view; -typedef basic_string_view u32string_view; -#endif - -}} // namespace nonstd::sv_lite - -// -// 24.4.6 Suffix for basic_string_view literals: -// - -#if nssv_HAVE_USER_DEFINED_LITERALS - -namespace nonstd { -nssv_inline_ns namespace literals { -nssv_inline_ns namespace string_view_literals { - -#if nssv_CONFIG_STD_SV_OPERATOR && nssv_HAVE_STD_DEFINED_LITERALS - -nssv_constexpr nonstd::sv_lite::string_view operator ""sv( const char* str, size_t len ) nssv_noexcept // (1) -{ - return nonstd::sv_lite::string_view{ str, len }; -} - -nssv_constexpr nonstd::sv_lite::u16string_view operator ""sv( const char16_t* str, size_t len ) nssv_noexcept // (2) -{ - return nonstd::sv_lite::u16string_view{ str, len }; -} - -nssv_constexpr nonstd::sv_lite::u32string_view operator ""sv( const char32_t* str, size_t len ) nssv_noexcept // (3) -{ - return nonstd::sv_lite::u32string_view{ str, len }; -} - -nssv_constexpr nonstd::sv_lite::wstring_view operator ""sv( const wchar_t* str, size_t len ) nssv_noexcept // (4) -{ - return nonstd::sv_lite::wstring_view{ str, len }; -} - -#endif // nssv_CONFIG_STD_SV_OPERATOR && nssv_HAVE_STD_DEFINED_LITERALS - -#if nssv_CONFIG_USR_SV_OPERATOR - -nssv_constexpr nonstd::sv_lite::string_view operator ""_sv( const char* str, size_t len ) nssv_noexcept // (1) -{ - return nonstd::sv_lite::string_view{ str, len }; -} - -nssv_constexpr nonstd::sv_lite::u16string_view operator ""_sv( const char16_t* str, size_t len ) nssv_noexcept // (2) -{ - return nonstd::sv_lite::u16string_view{ str, len }; -} - -nssv_constexpr nonstd::sv_lite::u32string_view operator ""_sv( const char32_t* str, size_t len ) nssv_noexcept // (3) -{ - return nonstd::sv_lite::u32string_view{ str, len }; -} - -nssv_constexpr nonstd::sv_lite::wstring_view operator ""_sv( const wchar_t* str, size_t len ) nssv_noexcept // (4) -{ - return nonstd::sv_lite::wstring_view{ str, len }; -} - -#endif // nssv_CONFIG_USR_SV_OPERATOR - -}}} // namespace nonstd::literals::string_view_literals - -#endif - -// -// Extensions for std::string: -// - -#if nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS - -namespace nonstd { -namespace sv_lite { - -// Exclude MSVC 14 (19.00): it yields ambiguous to_string(): - -#if nssv_CPP11_OR_GREATER && nssv_COMPILER_MSVC_VERSION != 140 - -template< class CharT, class Traits, class Allocator = std::allocator > -std::basic_string -to_string( basic_string_view v, Allocator const & a = Allocator() ) -{ - return std::basic_string( v.begin(), v.end(), a ); -} - -#else - -template< class CharT, class Traits > -std::basic_string -to_string( basic_string_view v ) -{ - return std::basic_string( v.begin(), v.end() ); -} - -template< class CharT, class Traits, class Allocator > -std::basic_string -to_string( basic_string_view v, Allocator const & a ) -{ - return std::basic_string( v.begin(), v.end(), a ); -} - -#endif // nssv_CPP11_OR_GREATER - -template< class CharT, class Traits, class Allocator > -basic_string_view -to_string_view( std::basic_string const & s ) -{ - return basic_string_view( s.data(), s.size() ); -} - -}} // namespace nonstd::sv_lite - -#endif // nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS - -// -// make types and algorithms available in namespace nonstd: -// - -namespace nonstd { - -using sv_lite::basic_string_view; -using sv_lite::string_view; -using sv_lite::wstring_view; - -#if nssv_HAVE_WCHAR16_T -using sv_lite::u16string_view; -#endif -#if nssv_HAVE_WCHAR32_T -using sv_lite::u32string_view; -#endif - -// literal "sv" - -using sv_lite::operator==; -using sv_lite::operator!=; -using sv_lite::operator<; -using sv_lite::operator<=; -using sv_lite::operator>; -using sv_lite::operator>=; - -#if ! nssv_CONFIG_NO_STREAM_INSERTION -using sv_lite::operator<<; -#endif - -#if nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS -using sv_lite::to_string; -using sv_lite::to_string_view; -#endif - -} // namespace nonstd - -// 24.4.5 Hash support (C++11): - -// Note: The hash value of a string view object is equal to the hash value of -// the corresponding string object. - -#if nssv_HAVE_STD_HASH - -#include - -namespace std { - -template<> -struct hash< nonstd::string_view > -{ -public: - std::size_t operator()( nonstd::string_view v ) const nssv_noexcept - { - return std::hash()( std::string( v.data(), v.size() ) ); - } -}; - -template<> -struct hash< nonstd::wstring_view > -{ -public: - std::size_t operator()( nonstd::wstring_view v ) const nssv_noexcept - { - return std::hash()( std::wstring( v.data(), v.size() ) ); - } -}; - -template<> -struct hash< nonstd::u16string_view > -{ -public: - std::size_t operator()( nonstd::u16string_view v ) const nssv_noexcept - { - return std::hash()( std::u16string( v.data(), v.size() ) ); - } -}; - -template<> -struct hash< nonstd::u32string_view > -{ -public: - std::size_t operator()( nonstd::u32string_view v ) const nssv_noexcept - { - return std::hash()( std::u32string( v.data(), v.size() ) ); - } -}; - -} // namespace std - -#endif // nssv_HAVE_STD_HASH - -nssv_RESTORE_WARNINGS() - -#endif // nssv_HAVE_STD_STRING_VIEW -#endif // NONSTD_SV_LITE_H_INCLUDED -/* end file simdjson/nonstd/string_view.hpp */ -SIMDJSON_POP_DISABLE_WARNINGS - -namespace std { - using string_view = nonstd::string_view; -} -#endif // SIMDJSON_HAS_STRING_VIEW -#undef SIMDJSON_HAS_STRING_VIEW // We are not going to need this macro anymore. - -/// If EXPR is an error, returns it. -#define SIMDJSON_TRY(EXPR) { auto _err = (EXPR); if (_err) { return _err; } } - -// Unless the programmer has already set SIMDJSON_DEVELOPMENT_CHECKS, -// we want to set it under debug builds. We detect a debug build -// under Visual Studio when the _DEBUG macro is set. Under the other -// compilers, we use the fact that they define __OPTIMIZE__ whenever -// they allow optimizations. -// It is possible that this could miss some cases where SIMDJSON_DEVELOPMENT_CHECKS -// is helpful, but the programmer can set the macro SIMDJSON_DEVELOPMENT_CHECKS. -// It could also wrongly set SIMDJSON_DEVELOPMENT_CHECKS (e.g., if the programmer -// sets _DEBUG in a release build under Visual Studio, or if some compiler fails to -// set the __OPTIMIZE__ macro). -// We make it so that if NDEBUG is defined, then SIMDJSON_DEVELOPMENT_CHECKS -// is not defined, irrespective of the compiler. -// We recommend that users set NDEBUG in release builds, so that -// SIMDJSON_DEVELOPMENT_CHECKS is not defined in release builds by default, -// irrespective of the compiler. -#ifndef SIMDJSON_DEVELOPMENT_CHECKS -#ifdef _MSC_VER -// Visual Studio seems to set _DEBUG for debug builds. -// We set SIMDJSON_DEVELOPMENT_CHECKS to 1 if _DEBUG is defined -// and NDEBUG is not defined. -#if defined(_DEBUG) && !defined(NDEBUG) -#define SIMDJSON_DEVELOPMENT_CHECKS 1 -#endif // _DEBUG -#else // _MSC_VER -// All other compilers appear to set __OPTIMIZE__ to a positive integer -// when the compiler is optimizing. -// We only set SIMDJSON_DEVELOPMENT_CHECKS if both __OPTIMIZE__ -// and NDEBUG are not defined. -// We recognize _DEBUG as overriding __OPTIMIZE__ so that if both -// __OPTIMIZE__ and _DEBUG are defined, we still set SIMDJSON_DEVELOPMENT_CHECKS. -#if ((!defined(__OPTIMIZE__) || defined(_DEBUG)) && !defined(NDEBUG)) -#define SIMDJSON_DEVELOPMENT_CHECKS 1 -#endif // __OPTIMIZE__ -#endif // _MSC_VER -#endif // SIMDJSON_DEVELOPMENT_CHECKS - -// The SIMDJSON_CHECK_EOF macro is a feature flag for the "don't require padding" -// feature. - -#if SIMDJSON_CPLUSPLUS17 -// if we have C++, then fallthrough is a default attribute -# define simdjson_fallthrough [[fallthrough]] -// check if we have __attribute__ support -#elif defined(__has_attribute) -// check if we have the __fallthrough__ attribute -#if __has_attribute(__fallthrough__) -// we are good to go: -# define simdjson_fallthrough __attribute__((__fallthrough__)) -#endif // __has_attribute(__fallthrough__) -#endif // SIMDJSON_CPLUSPLUS17 -// on some systems, we simply do not have support for fallthrough, so use a default: -#ifndef simdjson_fallthrough -# define simdjson_fallthrough do {} while (0) /* fallthrough */ -#endif // simdjson_fallthrough - -#if SIMDJSON_DEVELOPMENT_CHECKS -#define SIMDJSON_DEVELOPMENT_ASSERT(expr) do { assert ((expr)); } while (0) -#else -#define SIMDJSON_DEVELOPMENT_ASSERT(expr) do { } while (0) -#endif - -#ifndef SIMDJSON_UTF8VALIDATION -#define SIMDJSON_UTF8VALIDATION 1 -#endif - -#ifdef __has_include -// How do we detect that a compiler supports vbmi2? -// For sure if the following header is found, we are ok? -#if __has_include() -#define SIMDJSON_COMPILER_SUPPORTS_VBMI2 1 -#endif -#endif - -#ifdef _MSC_VER -#if _MSC_VER >= 1920 -// Visual Studio 2019 and up support VBMI2 under x64 even if the header -// avx512vbmi2intrin.h is not found. -#define SIMDJSON_COMPILER_SUPPORTS_VBMI2 1 -#endif -#endif - -// By default, we allow AVX512. -#ifndef SIMDJSON_AVX512_ALLOWED -#define SIMDJSON_AVX512_ALLOWED 1 -#endif - - -#ifndef __has_cpp_attribute -#define simdjson_lifetime_bound -#elif __has_cpp_attribute(msvc::lifetimebound) -#define simdjson_lifetime_bound [[msvc::lifetimebound]] -#elif __has_cpp_attribute(clang::lifetimebound) -#define simdjson_lifetime_bound [[clang::lifetimebound]] -#elif __has_cpp_attribute(lifetimebound) -#define simdjson_lifetime_bound [[lifetimebound]] -#else -#define simdjson_lifetime_bound -#endif -#endif // SIMDJSON_COMMON_DEFS_H -/* end file simdjson/common_defs.h */ -/* skipped duplicate #include "simdjson/compiler_check.h" */ -/* including simdjson/error.h: #include "simdjson/error.h" */ -/* begin file simdjson/error.h */ -#ifndef SIMDJSON_ERROR_H -#define SIMDJSON_ERROR_H - -/* skipped duplicate #include "simdjson/base.h" */ - -#include -#include - -namespace simdjson { - -/** - * All possible errors returned by simdjson. These error codes are subject to change - * and not all simdjson kernel returns the same error code given the same input: it is not - * well defined which error a given input should produce. - * - * Only SUCCESS evaluates to false as a Boolean. All other error codes will evaluate - * to true as a Boolean. - */ -enum error_code { - SUCCESS = 0, ///< No error - CAPACITY, ///< This parser can't support a document that big - MEMALLOC, ///< Error allocating memory, most likely out of memory - TAPE_ERROR, ///< Something went wrong, this is a generic error. Fatal/unrecoverable error. - DEPTH_ERROR, ///< Your document exceeds the user-specified depth limitation - STRING_ERROR, ///< Problem while parsing a string - T_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 't' - F_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'f' - N_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'n' - NUMBER_ERROR, ///< Problem while parsing a number - BIGINT_ERROR, ///< The integer value exceeds 64 bits - UTF8_ERROR, ///< the input is not valid UTF-8 - UNINITIALIZED, ///< unknown error, or uninitialized document - EMPTY, ///< no structural element found - UNESCAPED_CHARS, ///< found unescaped characters in a string. - UNCLOSED_STRING, ///< missing quote at the end - UNSUPPORTED_ARCHITECTURE, ///< unsupported architecture - INCORRECT_TYPE, ///< JSON element has a different type than user expected - NUMBER_OUT_OF_RANGE, ///< JSON number does not fit in 64 bits - INDEX_OUT_OF_BOUNDS, ///< JSON array index too large - NO_SUCH_FIELD, ///< JSON field not found in object - IO_ERROR, ///< Error reading a file - INVALID_JSON_POINTER, ///< Invalid JSON pointer syntax - INVALID_URI_FRAGMENT, ///< Invalid URI fragment - UNEXPECTED_ERROR, ///< indicative of a bug in simdjson - PARSER_IN_USE, ///< parser is already in use. - OUT_OF_ORDER_ITERATION, ///< tried to iterate an array or object out of order (checked when SIMDJSON_DEVELOPMENT_CHECKS=1) - INSUFFICIENT_PADDING, ///< The JSON doesn't have enough padding for simdjson to safely parse it. - INCOMPLETE_ARRAY_OR_OBJECT, ///< The document ends early. Fatal/unrecoverable error. - SCALAR_DOCUMENT_AS_VALUE, ///< A scalar document is treated as a value. - OUT_OF_BOUNDS, ///< Attempted to access location outside of document. - TRAILING_CONTENT, ///< Unexpected trailing content in the JSON input - OUT_OF_CAPACITY, ///< The capacity was exceeded, we cannot allocate enough memory. - UNKNOWN_FIELD, ///< JSON field does not map to any member of the target (see simdjson::deny_unknown_fields) - NUM_ERROR_CODES ///< Placeholder for end of error code list. -}; - -/** - * Some errors are fatal and invalidate the document. This function returns true if the - * error is fatal. It returns true for TAPE_ERROR and INCOMPLETE_ARRAY_OR_OBJECT. - * Once a fatal error is encountered, the on-demand document is no longer valid and - * processing should stop. - */ - inline bool is_fatal(error_code error) noexcept; - -/** - * It is the convention throughout the code that the macro SIMDJSON_DEVELOPMENT_CHECKS determines whether - * we check for OUT_OF_ORDER_ITERATION. The logic behind it is that these errors only occurs when the code - * that was written while breaking some simdjson::ondemand requirement. They should not occur in released - * code after these issues were fixed. - */ - -/** - * Get the error message for the given error code. - * - * dom::parser parser; - * dom::element doc; - * auto error = parser.parse("foo",3).get(doc); - * if (error) { printf("Error: %s\n", error_message(error)); } - * - * @return The error message. - */ -inline const char *error_message(error_code error) noexcept; - -/** - * Write the error message to the output stream - */ -inline std::ostream& operator<<(std::ostream& out, error_code error) noexcept; - -/** - * Exception thrown when an exception-supporting simdjson method is called - */ -struct simdjson_error : public std::exception { - /** - * Create an exception from a simdjson error code. - * @param error The error code - */ - simdjson_error(error_code error) noexcept : _error{error} { } - /** The error message */ - const char *what() const noexcept override { return error_message(error()); } - /** The error code */ - error_code error() const noexcept { return _error; } -private: - /** The error code that was used */ - error_code _error; -}; - -namespace internal { - -/** - * The result of a simdjson operation that could fail. - * - * IMPORTANT: For the ondemand API, we use implementation_simdjson_result_base as a base class - * to avoid some compilation issue. Thus, if you modify this class, please ensure that the ondemand - * implementation_simdjson_result_base is also modified. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::simdjson_result_base { - * simdjson_result() noexcept : internal::simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct simdjson_result_base : protected std::pair { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline simdjson_result_base() noexcept; - - /** - * Create a new error result. - */ - simdjson_inline simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_inline bool has_value() const noexcept; -#if SIMDJSON_EXCEPTIONS - - /** - * Dereference operator to access the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - * We discourage the use of value_unsafe(). - * - * The recommended pattern is: - * - * T value; // where T is the type - * auto error = result.get(value); - * if (error) { - * // handle error - * } - * - * Or you may call 'value()' which will raise an exception - * in case of error: - * - * T value = result.value(); - */ - simdjson_inline const T& value_unsafe() const& noexcept; - - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - * We discourage the use of value_unsafe(). - * - * The recommended pattern is: - * - * T value; // where T is the type - * auto error = result.get(value); - * if (error) { - * // handle error, return, exit, abort - * } else { - * // use value here. - * } - * - * Or you may call 'value()' which will raise an exception - * in case of error: - * - * T value = result.value(); - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; -}; // struct simdjson_result_base - -} // namespace internal - -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - */ -template -struct simdjson_result : public internal::simdjson_result_base { - - /** - * @private Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline simdjson_result() noexcept; - /** - * @private Create a new successful result. - */ - simdjson_inline simdjson_result(T &&value) noexcept; - /** - * @private Create a new error result. - */ - simdjson_inline simdjson_result(error_code error_code) noexcept; - /** - * @private Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline simdjson_result(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * Copy the value to a provided std::string, only enabled for std::string_view. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - template - simdjson_warn_unused simdjson_inline error_code get(std::string &value) && noexcept { - static_assert(std::is_same::value, "SFINAE"); - std::string_view v; - error_code error = std::forward>(*this).get(v); - if (!error) { - value.assign(v.data(), v.size()); - } - return error; - } - - /** - * The error. - */ - simdjson_inline error_code error() const noexcept; - - - -#if SIMDJSON_EXCEPTIONS - using internal::simdjson_result_base::operator*; - using internal::simdjson_result_base::operator->; - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; -}; // struct simdjson_result - -#if SIMDJSON_EXCEPTIONS - -template -inline std::ostream& operator<<(std::ostream& out, simdjson_result value) { return out << value.value(); } -#endif // SIMDJSON_EXCEPTIONS - -#ifndef SIMDJSON_DISABLE_DEPRECATED_API -/** - * @deprecated This is an alias and will be removed, use error_code instead - */ -using ErrorValues [[deprecated("This is an alias and will be removed, use error_code instead")]] = error_code; - -/** - * @deprecated Error codes should be stored and returned as `error_code`, use `error_message()` instead. - */ -[[deprecated("Error codes should be stored and returned as `error_code`, use `error_message()` instead.")]] -inline const std::string error_message(int error) noexcept; -#endif // SIMDJSON_DISABLE_DEPRECATED_API -} // namespace simdjson - -#endif // SIMDJSON_ERROR_H -/* end file simdjson/error.h */ -/* skipped duplicate #include "simdjson/portability.h" */ -/* including simdjson/concepts.h: #include "simdjson/concepts.h" */ -/* begin file simdjson/concepts.h */ -#ifndef SIMDJSON_CONCEPTS_H -#define SIMDJSON_CONCEPTS_H -#if SIMDJSON_SUPPORTS_CONCEPTS - -#include -#include -#include - -namespace simdjson { -namespace concepts { - -namespace details { -#define SIMDJSON_IMPL_CONCEPT(name, method) \ - template \ - concept supports_##name = !std::is_const_v && requires { \ - typename std::remove_cvref_t::value_type; \ - requires requires(typename std::remove_cvref_t::value_type &&val, \ - T obj) { \ - obj.method(std::move(val)); \ - requires !requires { obj = std::move(val); }; \ - }; \ - }; - -SIMDJSON_IMPL_CONCEPT(emplace_back, emplace_back) -SIMDJSON_IMPL_CONCEPT(emplace, emplace) -SIMDJSON_IMPL_CONCEPT(push_back, push_back) -SIMDJSON_IMPL_CONCEPT(add, add) -SIMDJSON_IMPL_CONCEPT(push, push) -SIMDJSON_IMPL_CONCEPT(append, append) -SIMDJSON_IMPL_CONCEPT(insert, insert) -SIMDJSON_IMPL_CONCEPT(op_append, operator+=) - -#undef SIMDJSON_IMPL_CONCEPT -} // namespace details - -template -concept is_pair = requires { typename T::first_type; typename T::second_type; } && - std::same_as>; -template -concept string_view_like = std::is_convertible_v && - !std::is_convertible_v; - -template -concept constructible_from_string_view = std::is_constructible_v - && !std::is_same_v - && std::is_default_constructible_v; - -#if SIMDJSON_SUPPORTS_CHAR8_T -/** - * A C++20 char8_t string type such as std::u8string. Such types cannot be built - * from a std::string_view (the character types differ), so they need their own - * deserialization path, going through the u8 string accessors. - */ -template -concept constructible_from_u8string_view = std::is_constructible_v - && !std::is_same_v - && !std::is_constructible_v - && std::is_default_constructible_v; -#endif // SIMDJSON_SUPPORTS_CHAR8_T - -template -concept string_view_keyed_map = string_view_like - && requires(std::remove_cvref_t& m, typename M::key_type sv, typename M::mapped_type v) { - { m.emplace(sv, v) } -> std::same_as>; -}; - -/// Check if T is a container that we can append to, including: -/// std::vector, std::deque, std::list, std::string, ... -template -concept appendable_containers = - (details::supports_emplace_back || details::supports_emplace || - details::supports_push_back || details::supports_push || - details::supports_add || details::supports_append || - details::supports_insert) && !string_view_keyed_map; - -/// Insert into the container however possible -template -constexpr decltype(auto) emplace_one(T &vec, Args &&...args) { - if constexpr (details::supports_emplace_back) { - return vec.emplace_back(std::forward(args)...); - } else if constexpr (details::supports_emplace) { - return vec.emplace(std::forward(args)...); - } else if constexpr (details::supports_push_back) { - return vec.push_back(std::forward(args)...); - } else if constexpr (details::supports_push) { - return vec.push(std::forward(args)...); - } else if constexpr (details::supports_add) { - return vec.add(std::forward(args)...); - } else if constexpr (details::supports_append) { - return vec.append(std::forward(args)...); - } else if constexpr (details::supports_insert) { - return vec.insert(std::forward(args)...); - } else if constexpr (details::supports_op_append && sizeof...(Args) == 1) { - return vec.operator+=(std::forward(args)...); - } else { - static_assert(!sizeof(T *), - "We don't know how to add things to this container"); - } -} - -/// This checks if the container will return a reference to the newly added -/// element after an insert which for example `std::vector::emplace_back` does -/// since C++17; this will allow some optimizations. -template -concept returns_reference = appendable_containers && requires { - typename std::remove_cvref_t::reference; - requires requires(typename std::remove_cvref_t::value_type &&val, T obj) { - { - emplace_one(obj, std::move(val)) - } -> std::same_as::reference>; - }; -}; - -template -concept smart_pointer = requires(std::remove_cvref_t ptr) { - // Check if T has a member type named element_type - typename std::remove_cvref_t::element_type; - - // Check if T has a get() member function - { - ptr.get() - } -> std::same_as::element_type *>; - - // Check if T can be dereferenced - { *ptr } -> std::same_as::element_type &>; -}; - -template -concept optional_type = requires(std::remove_cvref_t obj) { - typename std::remove_cvref_t::value_type; - { obj.value() } -> std::same_as::value_type&>; - requires requires(typename std::remove_cvref_t::value_type &&val) { - obj.emplace(std::move(val)); - { - obj.value_or(val) - } -> std::convertible_to::value_type>; - }; - { static_cast(obj) } -> std::same_as; // convertible to bool - { obj.reset() } noexcept -> std::same_as; -}; - - - -// Types we serialize as JSON strings (not as containers) -template -concept string_like = - std::is_same_v, std::string> || - std::is_same_v, std::string_view> || - std::is_same_v, const char*> || - std::is_same_v, char*>; - -// Concept that checks if a type is a container but not a string (because -// strings handling must be handled differently) -// Now uses iterator-based approach for broader container support -// -// Optional types are excluded on purpose. Since C++26 (P3168), std::optional -// is itself a range, so without the exclusion an std::optional would match -// both this concept and optional_type, making the container and the optional -// overloads of atom()/append() ambiguous. See issue 2827. -template -concept container_but_not_string = - std::ranges::input_range && !string_like && !concepts::string_view_keyed_map - && !concepts::optional_type; - - - -// Concept: Indexable container that is not a string or associative container -// Accepts: std::vector, std::array, std::deque (have operator[], value_type, not string_like) -// Rejects: std::string (string_like), std::list (no operator[]), std::map (has key_type) -template -concept indexable_container = requires { - typename Container::value_type; - requires !concepts::string_like; - requires !requires { typename Container::key_type; }; // Reject maps/sets - requires requires(Container& c, std::size_t i) { - { c[i] } -> std::convertible_to; - }; -}; - - -// Variable template to use with std::meta::substitute -template -constexpr bool indexable_container_v = indexable_container; - -} // namespace concepts - - -/** - * We use tag_invoke as our customization point mechanism. - */ -template -concept tag_invocable = requires(Tag tag, Args... args) { - tag_invoke(std::forward(tag), std::forward(args)...); -}; - -template -concept nothrow_tag_invocable = - tag_invocable && requires(Tag tag, Args... args) { - { - tag_invoke(std::forward(tag), std::forward(args)...) - } noexcept; - }; - -} // namespace simdjson -#endif // SIMDJSON_SUPPORTS_CONCEPTS -#endif // SIMDJSON_CONCEPTS_H -/* end file simdjson/concepts.h */ -/* including simdjson/constevalutil.h: #include "simdjson/constevalutil.h" */ -/* begin file simdjson/constevalutil.h */ -#ifndef SIMDJSON_CONSTEVALUTIL_H -#define SIMDJSON_CONSTEVALUTIL_H - -#include -#include -#include - -namespace simdjson { -namespace constevalutil { -#if SIMDJSON_CONSTEVAL - -constexpr static std::array json_quotable_character = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; -constexpr static std::array control_chars = { - "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", - "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r", - "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014", - "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b", - "\\u001c", "\\u001d", "\\u001e", "\\u001f"}; -// unoptimized, meant for compile-time execution -consteval std::string consteval_to_quoted_escaped(std::string_view input) { - std::string out = "\""; - for (char c : input) { - if (json_quotable_character[uint8_t(c)]) { - if (c == '"') { - out.append("\\\""); - } else if (c == '\\') { - out.append("\\\\"); - } else { - std::string_view v = control_chars[uint8_t(c)]; - out.append(v); - } - } else { - out.push_back(c); - } - } - out.push_back('"'); - return out; -} - -consteval std::size_t padded_key_length(std::size_t length) { - return (length + 15) / 16 * 16; -} -#endif // SIMDJSON_CONSTEVAL - - -#if SIMDJSON_SUPPORTS_CONCEPTS -template -struct fixed_string { - constexpr fixed_string(const char (&str)[N]) { - for (std::size_t i = 0; i < N; ++i) { - data[i] = str[i]; - } - } - constexpr fixed_string(const unsigned char (&str)[N]) { - for (std::size_t i = 0; i < N; ++i) { - data[i] = static_cast(str[i]); - } - } - char data[N]; - constexpr std::string_view view() const { return {data, N - 1}; } - constexpr size_t size() const { return N ; } - constexpr operator std::string_view() const { return view(); } - constexpr char operator[](std::size_t index) const { return data[index]; } - constexpr bool operator==(const fixed_string& other) const { - if (N != other.size()) { - return false; - } - for (std::size_t i = 0; i < N; ++i) { - if (data[i] != other.data[i]) { - return false; - } - } - return true; - } -}; -template -fixed_string(const char (&)[N]) -> fixed_string; - -template -inline constexpr bool is_fixed_string = false; -template -inline constexpr bool is_fixed_string> = true; - -template -struct string_constant { - static constexpr std::string_view value = str.view(); -}; - -#if SIMDJSON_CONSTEVAL -template -consteval std::size_t static_string_size() { - return Text::text().size(); -} -template -consteval std::size_t padded_static_string_size() { - return padded_key_length(static_string_size()); -} - -template -consteval auto make_static_string() { - std::array()> out{}; - std::string text = Text::text(); - // The tail past text.size() stays zero: it is the padding. - for (std::size_t i = 0; i < text.size(); ++i) { - out[i] = text[i]; - } - return out; -} - -template -inline constexpr auto static_string = make_static_string(); - -template -struct quoted_escaped_text { - static consteval std::string text() { return consteval_to_quoted_escaped(str.view()); } -}; -#endif // SIMDJSON_CONSTEVAL -#endif // SIMDJSON_SUPPORTS_CONCEPTS - -} // namespace constevalutil -} // namespace simdjson -#endif // SIMDJSON_CONSTEVALUTIL_H -/* end file simdjson/constevalutil.h */ - -#if SIMDJSON_SUPPORTS_CHAR8_T -#include -#include -#endif - -/** - * @brief The top level simdjson namespace, containing everything the library provides. - */ -namespace simdjson { - -SIMDJSON_PUSH_DISABLE_UNUSED_WARNINGS - -/** The maximum document size supported by simdjson. */ -constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF; -/** The maximum depth of nested objects and arrays supported by simdjson. - A depth of SIMDJSON_MAXSIZE_BYTES/2 is not reasonable and would be - adversarial, but it serves as an upper bound for validation purposes. */ -constexpr size_t SIMDJSON_MAX_DEPTH = SIMDJSON_MAXSIZE_BYTES/2; - -/** - * The amount of padding needed in a buffer to parse JSON. - * - * The input buf should be readable up to buf + SIMDJSON_PADDING - * this is a stopgap; there should be a better description of the - * main loop and its behavior that abstracts over this - * See https://github.com/simdjson/simdjson/issues/174 - */ -constexpr size_t SIMDJSON_PADDING = 64; - -/** - * By default, simdjson supports this many nested objects and arrays. - * - * This is the default for parser::max_depth(). - */ -constexpr size_t DEFAULT_MAX_DEPTH = 1024; - -SIMDJSON_POP_DISABLE_UNUSED_WARNINGS - -class implementation; -struct padded_string; -class padded_string_view; -enum class stage1_mode; - -/** - * Stream format for parse_many/iterate_many. - */ -enum class stream_format { - whitespace_delimited, ///< Whitespace-delimited JSON documents (default, includes NDJSON/JSONL) - json_sequence, ///< RFC 7464 JSON text sequences (RS-delimited) - comma_delimited, ///< Comma-separated JSON documents (e.g., `{...},{...},{...}`) - comma_delimited_array,///< A single JSON array whose elements are iterated as - ///< comma-separated documents (e.g., `[{...},{...},{...}]`). - ///< The parser strips the outer `[` / `]` plus any - ///< surrounding JSON whitespace (space, tab, LF, CR) - ///< and then behaves like `comma_delimited` over the - ///< remaining bytes. - newline_delimited ///< NDJSON/JSON Lines where each document occupies exactly - ///< one line: documents are separated by line feeds and no - ///< document contains a raw line feed. Same inputs as - ///< `whitespace_delimited`, but the stronger guarantee lets - ///< the parser find the end of a document without walking - ///< it. On ondemand `iterate_many`, an unread remainder may - ///< be skipped by jumping to the next line feed without - ///< structure-validating that remainder. Use - ///< `whitespace_delimited` if unsure. -}; - -namespace internal { - -template -class atomic_ptr; -class dom_parser_implementation; -class escape_json_string; -class tape_ref; -struct value128; -enum class tape_type; - -#if SIMDJSON_SUPPORTS_CHAR8_T -/** - * Reinterpret a UTF-8 string as a C++20 std::u8string_view. No byte is copied - * or modified: char8_t and char have the same size, representation and - * alignment. Every string that simdjson produces is valid UTF-8, so this is a - * lossless view over the very same memory. - * @private - */ -simdjson_inline std::u8string_view as_u8string_view(std::string_view v) noexcept { - return std::u8string_view(reinterpret_cast(v.data()), v.size()); -} -#endif // SIMDJSON_SUPPORTS_CHAR8_T - -/** - * Assign a UTF-8 string to a string-like receiver. The general case simply - * assigns the std::string_view: it covers std::string and any user type that - * can be assigned from a std::string_view. - * @private - */ -template -simdjson_inline void assign_utf8(string_type &receiver, std::string_view content) noexcept { - receiver = content; -} - -#if SIMDJSON_SUPPORTS_CHAR8_T -/** - * Assign a UTF-8 string to a char8_t-based string (e.g., std::u8string). This - * overload is more specialized than the general one, so overload resolution - * prefers it whenever the receiver holds char8_t. - * @private - */ -template -simdjson_inline void assign_utf8(std::basic_string &receiver, std::string_view content) noexcept { - receiver.assign(reinterpret_cast(content.data()), content.size()); -} - -/** - * Assign a UTF-8 string to a char8_t-based string view (e.g., std::u8string_view). - * @private - */ -template -simdjson_inline void assign_utf8(std::basic_string_view &receiver, std::string_view content) noexcept { - receiver = std::basic_string_view(reinterpret_cast(content.data()), content.size()); -} -#endif // SIMDJSON_SUPPORTS_CHAR8_T - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_BASE_H -/* end file simdjson/base.h */ - -#endif // SIMDJSON_SRC_BASE_H -/* end file base.h */ - -SIMDJSON_PUSH_DISABLE_UNUSED_WARNINGS - -/* including to_chars.cpp: #include */ -/* begin file to_chars.cpp */ -#ifndef SIMDJSON_SRC_TO_CHARS_CPP -#define SIMDJSON_SRC_TO_CHARS_CPP - -/* skipped duplicate #include */ - -#include -#include -#include -#include - -namespace simdjson { -namespace internal { -/*! -Implements the Dragonbox algorithm for binary to decimal floating-point -conversion (shortest round-trip representation of a double). - -The digit-generation core below is a self-contained port of Junekey Jeon's -reference "simple_dragonbox" implementation, specialized to IEEE-754 binary64 -and de-templated to match simdjson's style. Only the shortest-representation -path with the default (nearest, ties-to-even) rounding is kept. - -Dragonbox: https://github.com/jk-jeon/dragonbox -Copyright 2020-2025 Junekey Jeon (and contributors). - -The original is dual-licensed; this port is used under the terms of the -Boost Software License, Version 1.0 (https://www.boost.org/LICENSE_1_0.txt). - -For the algorithm itself see: -[1] Junekey Jeon, "Dragonbox: A New Floating-Point Binary-to-Decimal Conversion Algorithm" (2022). - -The shortest decimal digits produced here are then laid out into the familiar -printf("%g")-style text by format_buffer(), which is unchanged from the previous -Grisu2-based implementation, so the emitted strings are identical except that -Dragonbox always yields the (sometimes shorter) shortest representation. -*/ -namespace dtoa_impl { - -// 128-bit helpers (no compiler intrinsics, so the code stays portable). -struct uint128 { - std::uint64_t high; - std::uint64_t low; -}; - -inline std::uint64_t rotr64(std::uint64_t n, unsigned r) noexcept { - r &= 63; - return (n >> r) | (n << ((64 - r) & 63)); -} - -inline std::uint64_t umul64(std::uint32_t x, std::uint32_t y) noexcept { - return x * std::uint64_t(y); -} - -// 64x64 -> 128 bit multiplication. -inline uint128 umul128(std::uint64_t x, std::uint64_t y) noexcept { -#if defined(__SIZEOF_INT128__) - const __uint128_t p = static_cast<__uint128_t>(x)*y; - return {std::uint64_t(p>>64), std::uint64_t(p)}; -#else // using fallback on 32-bit targets and MSVC - const std::uint32_t a = std::uint32_t(x >> 32); - const std::uint32_t b = std::uint32_t(x); - const std::uint32_t c = std::uint32_t(y >> 32); - const std::uint32_t d = std::uint32_t(y); - - const std::uint64_t ac = umul64(a, c); - const std::uint64_t bc = umul64(b, c); - const std::uint64_t ad = umul64(a, d); - const std::uint64_t bd = umul64(b, d); - - const std::uint64_t intermediate = - (bd >> 32) + std::uint32_t(ad) + std::uint32_t(bc); - - return {ac + (intermediate >> 32) + (ad >> 32) + (bc >> 32), - (intermediate << 32) + std::uint32_t(bd)}; -#endif -} - -// High 64 bits of a 64x64 -> 128 bit multiplication. -inline std::uint64_t umul128_upper64(std::uint64_t x, std::uint64_t y) noexcept { -#if defined(__SIZEOF_INT128__) - return std::uint64_t((static_cast<__uint128_t>(x)*y)>>64); -#else // using fallback on 32-bit targets and MSVC - const std::uint32_t a = std::uint32_t(x >> 32); - const std::uint32_t b = std::uint32_t(x); - const std::uint32_t c = std::uint32_t(y >> 32); - const std::uint32_t d = std::uint32_t(y); - - const std::uint64_t ac = umul64(a, c); - const std::uint64_t bc = umul64(b, c); - const std::uint64_t ad = umul64(a, d); - const std::uint64_t bd = umul64(b, d); - - const std::uint64_t intermediate = - (bd >> 32) + std::uint32_t(ad) + std::uint32_t(bc); - - return ac + (intermediate >> 32) + (ad >> 32) + (bc >> 32); -#endif -} - -// Upper 128 bits of a 64 x 128 -> 192 bit multiplication. -inline uint128 umul192_upper128(std::uint64_t x, uint128 y) noexcept { - uint128 r = umul128(x, y.high); - const std::uint64_t add = umul128_upper64(x, y.low); - const std::uint64_t sum = r.low + add; - r.high += (sum < r.low) ? 1 : 0; - r.low = sum; - return r; -} - -// Lower 128 bits of a 64 x 128 -> 192 bit multiplication. -inline uint128 umul192_lower128(std::uint64_t x, uint128 y) noexcept { - const std::uint64_t high = x * y.high; - const uint128 high_low = umul128(x, y.low); - return {high + high_low.high, high_low.low}; -} - -// Integer log approximations (exact over the range of inputs we feed them). -inline int floor_log10_pow2(int e) noexcept { return (e * 315653) >> 20; } -inline int floor_log2_pow10(int e) noexcept { return (e * 1741647) >> 19; } -inline int floor_log10_pow2_minus_log10_4_over_3(int e) noexcept { - return (e * 631305 - 261663) >> 21; -} - -// Format constants for IEEE-754 binary64, plus the precomputed cache of powers of ten. -static constexpr int kappa = 2; -static constexpr int significand_bits = 52; -static constexpr int total_bits = 64; -static constexpr int min_exponent = -1022; -static constexpr int exponent_bias = -1023; -static constexpr int cache_min_k = -292; -static constexpr int big_divisor = 1000; // 10^(kappa + 1) -static constexpr int small_divisor = 100; // 10^kappa -static constexpr int case_shorter_interval_left_endpoint_lower_threshold = 2; -static constexpr int case_shorter_interval_left_endpoint_upper_threshold = 3; -static constexpr int shorter_interval_tie_lower_threshold = -77; -static constexpr int shorter_interval_tie_upper_threshold = -77; - -// cache[i] holds a 128-bit approximation of a power of ten; indexed by -// (-minus_k - cache_min_k). Taken verbatim from the Dragonbox reference. -static constexpr uint128 cache[619] = { - {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7b}, - {0x9faacf3df73609b1, 0x77b191618c54e9ad}, - {0xc795830d75038c1d, 0xd59df5b9ef6a2418}, - {0xf97ae3d0d2446f25, 0x4b0573286b44ad1e}, - {0x9becce62836ac577, 0x4ee367f9430aec33}, - {0xc2e801fb244576d5, 0x229c41f793cda740}, - {0xf3a20279ed56d48a, 0x6b43527578c11110}, - {0x9845418c345644d6, 0x830a13896b78aaaa}, - {0xbe5691ef416bd60c, 0x23cc986bc656d554}, - {0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa9}, - {0x94b3a202eb1c3f39, 0x7bf7d71432f3d6aa}, - {0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc54}, - {0xe858ad248f5c22c9, 0xd1b3400f8f9cff69}, - {0x91376c36d99995be, 0x23100809b9c21fa2}, - {0xb58547448ffffb2d, 0xabd40a0c2832a78b}, - {0xe2e69915b3fff9f9, 0x16c90c8f323f516d}, - {0x8dd01fad907ffc3b, 0xae3da7d97f6792e4}, - {0xb1442798f49ffb4a, 0x99cd11cfdf41779d}, - {0xdd95317f31c7fa1d, 0x40405643d711d584}, - {0x8a7d3eef7f1cfc52, 0x482835ea666b2573}, - {0xad1c8eab5ee43b66, 0xda3243650005eed0}, - {0xd863b256369d4a40, 0x90bed43e40076a83}, - {0x873e4f75e2224e68, 0x5a7744a6e804a292}, - {0xa90de3535aaae202, 0x711515d0a205cb37}, - {0xd3515c2831559a83, 0x0d5a5b44ca873e04}, - {0x8412d9991ed58091, 0xe858790afe9486c3}, - {0xa5178fff668ae0b6, 0x626e974dbe39a873}, - {0xce5d73ff402d98e3, 0xfb0a3d212dc81290}, - {0x80fa687f881c7f8e, 0x7ce66634bc9d0b9a}, - {0xa139029f6a239f72, 0x1c1fffc1ebc44e81}, - {0xc987434744ac874e, 0xa327ffb266b56221}, - {0xfbe9141915d7a922, 0x4bf1ff9f0062baa9}, - {0x9d71ac8fada6c9b5, 0x6f773fc3603db4aa}, - {0xc4ce17b399107c22, 0xcb550fb4384d21d4}, - {0xf6019da07f549b2b, 0x7e2a53a146606a49}, - {0x99c102844f94e0fb, 0x2eda7444cbfc426e}, - {0xc0314325637a1939, 0xfa911155fefb5309}, - {0xf03d93eebc589f88, 0x793555ab7eba27cb}, - {0x96267c7535b763b5, 0x4bc1558b2f3458df}, - {0xbbb01b9283253ca2, 0x9eb1aaedfb016f17}, - {0xea9c227723ee8bcb, 0x465e15a979c1cadd}, - {0x92a1958a7675175f, 0x0bfacd89ec191eca}, - {0xb749faed14125d36, 0xcef980ec671f667c}, - {0xe51c79a85916f484, 0x82b7e12780e7401b}, - {0x8f31cc0937ae58d2, 0xd1b2ecb8b0908811}, - {0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa16}, - {0xdfbdcece67006ac9, 0x67a791e093e1d49b}, - {0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e1}, - {0xaecc49914078536d, 0x58fae9f773886e19}, - {0xda7f5bf590966848, 0xaf39a475506a899f}, - {0x888f99797a5e012d, 0x6d8406c952429604}, - {0xaab37fd7d8f58178, 0xc8e5087ba6d33b84}, - {0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a65}, - {0x855c3be0a17fcd26, 0x5cf2eea09a550680}, - {0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481f}, - {0xd0601d8efc57b08b, 0xf13b94daf124da27}, - {0x823c12795db6ce57, 0x76c53d08d6b70859}, - {0xa2cb1717b52481ed, 0x54768c4b0c64ca6f}, - {0xcb7ddcdda26da268, 0xa9942f5dcf7dfd0a}, - {0xfe5d54150b090b02, 0xd3f93b35435d7c4d}, - {0x9efa548d26e5a6e1, 0xc47bc5014a1a6db0}, - {0xc6b8e9b0709f109a, 0x359ab6419ca1091c}, - {0xf867241c8cc6d4c0, 0xc30163d203c94b63}, - {0x9b407691d7fc44f8, 0x79e0de63425dcf1e}, - {0xc21094364dfb5636, 0x985915fc12f542e5}, - {0xf294b943e17a2bc4, 0x3e6f5b7b17b2939e}, - {0x979cf3ca6cec5b5a, 0xa705992ceecf9c43}, - {0xbd8430bd08277231, 0x50c6ff782a838354}, - {0xece53cec4a314ebd, 0xa4f8bf5635246429}, - {0x940f4613ae5ed136, 0x871b7795e136be9a}, - {0xb913179899f68584, 0x28e2557b59846e40}, - {0xe757dd7ec07426e5, 0x331aeada2fe589d0}, - {0x9096ea6f3848984f, 0x3ff0d2c85def7622}, - {0xb4bca50b065abe63, 0x0fed077a756b53aa}, - {0xe1ebce4dc7f16dfb, 0xd3e8495912c62895}, - {0x8d3360f09cf6e4bd, 0x64712dd7abbbd95d}, - {0xb080392cc4349dec, 0xbd8d794d96aacfb4}, - {0xdca04777f541c567, 0xecf0d7a0fc5583a1}, - {0x89e42caaf9491b60, 0xf41686c49db57245}, - {0xac5d37d5b79b6239, 0x311c2875c522ced6}, - {0xd77485cb25823ac7, 0x7d633293366b828c}, - {0x86a8d39ef77164bc, 0xae5dff9c02033198}, - {0xa8530886b54dbdeb, 0xd9f57f830283fdfd}, - {0xd267caa862a12d66, 0xd072df63c324fd7c}, - {0x8380dea93da4bc60, 0x4247cb9e59f71e6e}, - {0xa46116538d0deb78, 0x52d9be85f074e609}, - {0xcd795be870516656, 0x67902e276c921f8c}, - {0x806bd9714632dff6, 0x00ba1cd8a3db53b7}, - {0xa086cfcd97bf97f3, 0x80e8a40eccd228a5}, - {0xc8a883c0fdaf7df0, 0x6122cd128006b2ce}, - {0xfad2a4b13d1b5d6c, 0x796b805720085f82}, - {0x9cc3a6eec6311a63, 0xcbe3303674053bb1}, - {0xc3f490aa77bd60fc, 0xbedbfc4411068a9d}, - {0xf4f1b4d515acb93b, 0xee92fb5515482d45}, - {0x991711052d8bf3c5, 0x751bdd152d4d1c4b}, - {0xbf5cd54678eef0b6, 0xd262d45a78a0635e}, - {0xef340a98172aace4, 0x86fb897116c87c35}, - {0x9580869f0e7aac0e, 0xd45d35e6ae3d4da1}, - {0xbae0a846d2195712, 0x8974836059cca10a}, - {0xe998d258869facd7, 0x2bd1a438703fc94c}, - {0x91ff83775423cc06, 0x7b6306a34627ddd0}, - {0xb67f6455292cbf08, 0x1a3bc84c17b1d543}, - {0xe41f3d6a7377eeca, 0x20caba5f1d9e4a94}, - {0x8e938662882af53e, 0x547eb47b7282ee9d}, - {0xb23867fb2a35b28d, 0xe99e619a4f23aa44}, - {0xdec681f9f4c31f31, 0x6405fa00e2ec94d5}, - {0x8b3c113c38f9f37e, 0xde83bc408dd3dd05}, - {0xae0b158b4738705e, 0x9624ab50b148d446}, - {0xd98ddaee19068c76, 0x3badd624dd9b0958}, - {0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d7}, - {0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4d}, - {0xd47487cc8470652b, 0x7647c32000696720}, - {0x84c8d4dfd2c63f3b, 0x29ecd9f40041e074}, - {0xa5fb0a17c777cf09, 0xf468107100525891}, - {0xcf79cc9db955c2cc, 0x7182148d4066eeb5}, - {0x81ac1fe293d599bf, 0xc6f14cd848405531}, - {0xa21727db38cb002f, 0xb8ada00e5a506a7d}, - {0xca9cf1d206fdc03b, 0xa6d90811f0e4851d}, - {0xfd442e4688bd304a, 0x908f4a166d1da664}, - {0x9e4a9cec15763e2e, 0x9a598e4e043287ff}, - {0xc5dd44271ad3cdba, 0x40eff1e1853f29fe}, - {0xf7549530e188c128, 0xd12bee59e68ef47d}, - {0x9a94dd3e8cf578b9, 0x82bb74f8301958cf}, - {0xc13a148e3032d6e7, 0xe36a52363c1faf02}, - {0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac2}, - {0x96f5600f15a7b7e5, 0x29ab103a5ef8c0ba}, - {0xbcb2b812db11a5de, 0x7415d448f6b6f0e8}, - {0xebdf661791d60f56, 0x111b495b3464ad22}, - {0x936b9fcebb25c995, 0xcab10dd900beec35}, - {0xb84687c269ef3bfb, 0x3d5d514f40eea743}, - {0xe65829b3046b0afa, 0x0cb4a5a3112a5113}, - {0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ac}, - {0xb3f4e093db73a093, 0x59ed216765690f57}, - {0xe0f218b8d25088b8, 0x306869c13ec3532d}, - {0x8c974f7383725573, 0x1e414218c73a13fc}, - {0xafbd2350644eeacf, 0xe5d1929ef90898fb}, - {0xdbac6c247d62a583, 0xdf45f746b74abf3a}, - {0x894bc396ce5da772, 0x6b8bba8c328eb784}, - {0xab9eb47c81f5114f, 0x066ea92f3f326565}, - {0xd686619ba27255a2, 0xc80a537b0efefebe}, - {0x8613fd0145877585, 0xbd06742ce95f5f37}, - {0xa798fc4196e952e7, 0x2c48113823b73705}, - {0xd17f3b51fca3a7a0, 0xf75a15862ca504c6}, - {0x82ef85133de648c4, 0x9a984d73dbe722fc}, - {0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebbb}, - {0xcc963fee10b7d1b3, 0x318df905079926a9}, - {0xffbbcfe994e5c61f, 0xfdf17746497f7053}, - {0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa634}, - {0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc1}, - {0xf9bd690a1b68637b, 0x3dfdce7aa3c673b1}, - {0x9c1661a651213e2d, 0x06bea10ca65c084f}, - {0xc31bfa0fe5698db8, 0x486e494fcff30a63}, - {0xf3e2f893dec3f126, 0x5a89dba3c3efccfb}, - {0x986ddb5c6b3a76b7, 0xf89629465a75e01d}, - {0xbe89523386091465, 0xf6bbb397f1135824}, - {0xee2ba6c0678b597f, 0x746aa07ded582e2d}, - {0x94db483840b717ef, 0xa8c2a44eb4571cdd}, - {0xba121a4650e4ddeb, 0x92f34d62616ce414}, - {0xe896a0d7e51e1566, 0x77b020baf9c81d18}, - {0x915e2486ef32cd60, 0x0ace1474dc1d122f}, - {0xb5b5ada8aaff80b8, 0x0d819992132456bb}, - {0xe3231912d5bf60e6, 0x10e1fff697ed6c6a}, - {0x8df5efabc5979c8f, 0xca8d3ffa1ef463c2}, - {0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb3}, - {0xddd0467c64bce4a0, 0xac7cb3f6d05ddbdf}, - {0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96c}, - {0xad4ab7112eb3929d, 0x86c16c98d2c953c7}, - {0xd89d64d57a607744, 0xe871c7bf077ba8b8}, - {0x87625f056c7c4a8b, 0x11471cd764ad4973}, - {0xa93af6c6c79b5d2d, 0xd598e40d3dd89bd0}, - {0xd389b47879823479, 0x4aff1d108d4ec2c4}, - {0x843610cb4bf160cb, 0xcedf722a585139bb}, - {0xa54394fe1eedb8fe, 0xc2974eb4ee658829}, - {0xce947a3da6a9273e, 0x733d226229feea33}, - {0x811ccc668829b887, 0x0806357d5a3f5260}, - {0xa163ff802a3426a8, 0xca07c2dcb0cf26f8}, - {0xc9bcff6034c13052, 0xfc89b393dd02f0b6}, - {0xfc2c3f3841f17c67, 0xbbac2078d443ace3}, - {0x9d9ba7832936edc0, 0xd54b944b84aa4c0e}, - {0xc5029163f384a931, 0x0a9e795e65d4df12}, - {0xf64335bcf065d37d, 0x4d4617b5ff4a16d6}, - {0x99ea0196163fa42e, 0x504bced1bf8e4e46}, - {0xc06481fb9bcf8d39, 0xe45ec2862f71e1d7}, - {0xf07da27a82c37088, 0x5d767327bb4e5a4d}, - {0x964e858c91ba2655, 0x3a6a07f8d510f870}, - {0xbbe226efb628afea, 0x890489f70a55368c}, - {0xeadab0aba3b2dbe5, 0x2b45ac74ccea842f}, - {0x92c8ae6b464fc96f, 0x3b0b8bc90012929e}, - {0xb77ada0617e3bbcb, 0x09ce6ebb40173745}, - {0xe55990879ddcaabd, 0xcc420a6a101d0516}, - {0x8f57fa54c2a9eab6, 0x9fa946824a12232e}, - {0xb32df8e9f3546564, 0x47939822dc96abfa}, - {0xdff9772470297ebd, 0x59787e2b93bc56f8}, - {0x8bfbea76c619ef36, 0x57eb4edb3c55b65b}, - {0xaefae51477a06b03, 0xede622920b6b23f2}, - {0xdab99e59958885c4, 0xe95fab368e45ecee}, - {0x88b402f7fd75539b, 0x11dbcb0218ebb415}, - {0xaae103b5fcd2a881, 0xd652bdc29f26a11a}, - {0xd59944a37c0752a2, 0x4be76d3346f04960}, - {0x857fcae62d8493a5, 0x6f70a4400c562ddc}, - {0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb953}, - {0xd097ad07a71f26b2, 0x7e2000a41346a7a8}, - {0x825ecc24c873782f, 0x8ed400668c0c28c9}, - {0xa2f67f2dfa90563b, 0x728900802f0f32fb}, - {0xcbb41ef979346bca, 0x4f2b40a03ad2ffba}, - {0xfea126b7d78186bc, 0xe2f610c84987bfa9}, - {0x9f24b832e6b0f436, 0x0dd9ca7d2df4d7ca}, - {0xc6ede63fa05d3143, 0x91503d1c79720dbc}, - {0xf8a95fcf88747d94, 0x75a44c6397ce912b}, - {0x9b69dbe1b548ce7c, 0xc986afbe3ee11abb}, - {0xc24452da229b021b, 0xfbe85badce996169}, - {0xf2d56790ab41c2a2, 0xfae27299423fb9c4}, - {0x97c560ba6b0919a5, 0xdccd879fc967d41b}, - {0xbdb6b8e905cb600f, 0x5400e987bbc1c921}, - {0xed246723473e3813, 0x290123e9aab23b69}, - {0x9436c0760c86e30b, 0xf9a0b6720aaf6522}, - {0xb94470938fa89bce, 0xf808e40e8d5b3e6a}, - {0xe7958cb87392c2c2, 0xb60b1d1230b20e05}, - {0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c3}, - {0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af4}, - {0xe2280b6c20dd5232, 0x25c6da63c38de1b1}, - {0x8d590723948a535f, 0x579c487e5a38ad0f}, - {0xb0af48ec79ace837, 0x2d835a9df0c6d852}, - {0xdcdb1b2798182244, 0xf8e431456cf88e66}, - {0x8a08f0f8bf0f156b, 0x1b8e9ecb641b5900}, - {0xac8b2d36eed2dac5, 0xe272467e3d222f40}, - {0xd7adf884aa879177, 0x5b0ed81dcc6abb10}, - {0x86ccbb52ea94baea, 0x98e947129fc2b4ea}, - {0xa87fea27a539e9a5, 0x3f2398d747b36225}, - {0xd29fe4b18e88640e, 0x8eec7f0d19a03aae}, - {0x83a3eeeef9153e89, 0x1953cf68300424ad}, - {0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd8}, - {0xcdb02555653131b6, 0x3792f412cb06794e}, - {0x808e17555f3ebf11, 0xe2bbd88bbee40bd1}, - {0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec5}, - {0xc8de047564d20a8b, 0xf245825a5a445276}, - {0xfb158592be068d2e, 0xeed6e2f0f0d56713}, - {0x9ced737bb6c4183d, 0x55464dd69685606c}, - {0xc428d05aa4751e4c, 0xaa97e14c3c26b887}, - {0xf53304714d9265df, 0xd53dd99f4b3066a9}, - {0x993fe2c6d07b7fab, 0xe546a8038efe402a}, - {0xbf8fdb78849a5f96, 0xde98520472bdd034}, - {0xef73d256a5c0f77c, 0x963e66858f6d4441}, - {0x95a8637627989aad, 0xdde7001379a44aa9}, - {0xbb127c53b17ec159, 0x5560c018580d5d53}, - {0xe9d71b689dde71af, 0xaab8f01e6e10b4a7}, - {0x9226712162ab070d, 0xcab3961304ca70e9}, - {0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d23}, - {0xe45c10c42a2b3b05, 0x8cb89a7db77c506b}, - {0x8eb98a7a9a5b04e3, 0x77f3608e92adb243}, - {0xb267ed1940f1c61c, 0x55f038b237591ed4}, - {0xdf01e85f912e37a3, 0x6b6c46dec52f6689}, - {0x8b61313bbabce2c6, 0x2323ac4b3b3da016}, - {0xae397d8aa96c1b77, 0xabec975e0a0d081b}, - {0xd9c7dced53c72255, 0x96e7bd358c904a22}, - {0x881cea14545c7575, 0x7e50d64177da2e55}, - {0xaa242499697392d2, 0xdde50bd1d5d0b9ea}, - {0xd4ad2dbfc3d07787, 0x955e4ec64b44e865}, - {0x84ec3c97da624ab4, 0xbd5af13bef0b113f}, - {0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58f}, - {0xcfb11ead453994ba, 0x67de18eda5814af3}, - {0x81ceb32c4b43fcf4, 0x80eacf948770ced8}, - {0xa2425ff75e14fc31, 0xa1258379a94d028e}, - {0xcad2f7f5359a3b3e, 0x096ee45813a04331}, - {0xfd87b5f28300ca0d, 0x8bca9d6e188853fd}, - {0x9e74d1b791e07e48, 0x775ea264cf55347e}, - {0xc612062576589dda, 0x95364afe032a819e}, - {0xf79687aed3eec551, 0x3a83ddbd83f52205}, - {0x9abe14cd44753b52, 0xc4926a9672793543}, - {0xc16d9a0095928a27, 0x75b7053c0f178294}, - {0xf1c90080baf72cb1, 0x5324c68b12dd6339}, - {0x971da05074da7bee, 0xd3f6fc16ebca5e04}, - {0xbce5086492111aea, 0x88f4bb1ca6bcf585}, - {0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6}, - {0x9392ee8e921d5d07, 0x3aff322e62439fd0}, - {0xb877aa3236a4b449, 0x09befeb9fad487c3}, - {0xe69594bec44de15b, 0x4c2ebe687989a9b4}, - {0x901d7cf73ab0acd9, 0x0f9d37014bf60a11}, - {0xb424dc35095cd80f, 0x538484c19ef38c95}, - {0xe12e13424bb40e13, 0x2865a5f206b06fba}, - {0x8cbccc096f5088cb, 0xf93f87b7442e45d4}, - {0xafebff0bcb24aafe, 0xf78f69a51539d749}, - {0xdbe6fecebdedd5be, 0xb573440e5a884d1c}, - {0x89705f4136b4a597, 0x31680a88f8953031}, - {0xabcc77118461cefc, 0xfdc20d2b36ba7c3e}, - {0xd6bf94d5e57a42bc, 0x3d32907604691b4d}, - {0x8637bd05af6c69b5, 0xa63f9a49c2c1b110}, - {0xa7c5ac471b478423, 0x0fcf80dc33721d54}, - {0xd1b71758e219652b, 0xd3c36113404ea4a9}, - {0x83126e978d4fdf3b, 0x645a1cac083126ea}, - {0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4}, - {0xcccccccccccccccc, 0xcccccccccccccccd}, - {0x8000000000000000, 0x0000000000000000}, - {0xa000000000000000, 0x0000000000000000}, - {0xc800000000000000, 0x0000000000000000}, - {0xfa00000000000000, 0x0000000000000000}, - {0x9c40000000000000, 0x0000000000000000}, - {0xc350000000000000, 0x0000000000000000}, - {0xf424000000000000, 0x0000000000000000}, - {0x9896800000000000, 0x0000000000000000}, - {0xbebc200000000000, 0x0000000000000000}, - {0xee6b280000000000, 0x0000000000000000}, - {0x9502f90000000000, 0x0000000000000000}, - {0xba43b74000000000, 0x0000000000000000}, - {0xe8d4a51000000000, 0x0000000000000000}, - {0x9184e72a00000000, 0x0000000000000000}, - {0xb5e620f480000000, 0x0000000000000000}, - {0xe35fa931a0000000, 0x0000000000000000}, - {0x8e1bc9bf04000000, 0x0000000000000000}, - {0xb1a2bc2ec5000000, 0x0000000000000000}, - {0xde0b6b3a76400000, 0x0000000000000000}, - {0x8ac7230489e80000, 0x0000000000000000}, - {0xad78ebc5ac620000, 0x0000000000000000}, - {0xd8d726b7177a8000, 0x0000000000000000}, - {0x878678326eac9000, 0x0000000000000000}, - {0xa968163f0a57b400, 0x0000000000000000}, - {0xd3c21bcecceda100, 0x0000000000000000}, - {0x84595161401484a0, 0x0000000000000000}, - {0xa56fa5b99019a5c8, 0x0000000000000000}, - {0xcecb8f27f4200f3a, 0x0000000000000000}, - {0x813f3978f8940984, 0x4000000000000000}, - {0xa18f07d736b90be5, 0x5000000000000000}, - {0xc9f2c9cd04674ede, 0xa400000000000000}, - {0xfc6f7c4045812296, 0x4d00000000000000}, - {0x9dc5ada82b70b59d, 0xf020000000000000}, - {0xc5371912364ce305, 0x6c28000000000000}, - {0xf684df56c3e01bc6, 0xc732000000000000}, - {0x9a130b963a6c115c, 0x3c7f400000000000}, - {0xc097ce7bc90715b3, 0x4b9f100000000000}, - {0xf0bdc21abb48db20, 0x1e86d40000000000}, - {0x96769950b50d88f4, 0x1314448000000000}, - {0xbc143fa4e250eb31, 0x17d955a000000000}, - {0xeb194f8e1ae525fd, 0x5dcfab0800000000}, - {0x92efd1b8d0cf37be, 0x5aa1cae500000000}, - {0xb7abc627050305ad, 0xf14a3d9e40000000}, - {0xe596b7b0c643c719, 0x6d9ccd05d0000000}, - {0x8f7e32ce7bea5c6f, 0xe4820023a2000000}, - {0xb35dbf821ae4f38b, 0xdda2802c8a800000}, - {0xe0352f62a19e306e, 0xd50b2037ad200000}, - {0x8c213d9da502de45, 0x4526f422cc340000}, - {0xaf298d050e4395d6, 0x9670b12b7f410000}, - {0xdaf3f04651d47b4c, 0x3c0cdd765f114000}, - {0x88d8762bf324cd0f, 0xa5880a69fb6ac800}, - {0xab0e93b6efee0053, 0x8eea0d047a457a00}, - {0xd5d238a4abe98068, 0x72a4904598d6d880}, - {0x85a36366eb71f041, 0x47a6da2b7f864750}, - {0xa70c3c40a64e6c51, 0x999090b65f67d924}, - {0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d}, - {0x82818f1281ed449f, 0xbff8f10e7a8921a5}, - {0xa321f2d7226895c7, 0xaff72d52192b6a0e}, - {0xcbea6f8ceb02bb39, 0x9bf4f8a69f764491}, - {0xfee50b7025c36a08, 0x02f236d04753d5b5}, - {0x9f4f2726179a2245, 0x01d762422c946591}, - {0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef6}, - {0xf8ebad2b84e0d58b, 0xd2e0898765a7deb3}, - {0x9b934c3b330c8577, 0x63cc55f49f88eb30}, - {0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fc}, - {0xf316271c7fc3908a, 0x8bef464e3945ef7b}, - {0x97edd871cfda3a56, 0x97758bf0e3cbb5ad}, - {0xbde94e8e43d0c8ec, 0x3d52eeed1cbea318}, - {0xed63a231d4c4fb27, 0x4ca7aaa863ee4bde}, - {0x945e455f24fb1cf8, 0x8fe8caa93e74ef6b}, - {0xb975d6b6ee39e436, 0xb3e2fd538e122b45}, - {0xe7d34c64a9c85d44, 0x60dbbca87196b617}, - {0x90e40fbeea1d3a4a, 0xbc8955e946fe31ce}, - {0xb51d13aea4a488dd, 0x6babab6398bdbe42}, - {0xe264589a4dcdab14, 0xc696963c7eed2dd2}, - {0x8d7eb76070a08aec, 0xfc1e1de5cf543ca3}, - {0xb0de65388cc8ada8, 0x3b25a55f43294bcc}, - {0xdd15fe86affad912, 0x49ef0eb713f39ebf}, - {0x8a2dbf142dfcc7ab, 0x6e3569326c784338}, - {0xacb92ed9397bf996, 0x49c2c37f07965405}, - {0xd7e77a8f87daf7fb, 0xdc33745ec97be907}, - {0x86f0ac99b4e8dafd, 0x69a028bb3ded71a4}, - {0xa8acd7c0222311bc, 0xc40832ea0d68ce0d}, - {0xd2d80db02aabd62b, 0xf50a3fa490c30191}, - {0x83c7088e1aab65db, 0x792667c6da79e0fb}, - {0xa4b8cab1a1563f52, 0x577001b891185939}, - {0xcde6fd5e09abcf26, 0xed4c0226b55e6f87}, - {0x80b05e5ac60b6178, 0x544f8158315b05b5}, - {0xa0dc75f1778e39d6, 0x696361ae3db1c722}, - {0xc913936dd571c84c, 0x03bc3a19cd1e38ea}, - {0xfb5878494ace3a5f, 0x04ab48a04065c724}, - {0x9d174b2dcec0e47b, 0x62eb0d64283f9c77}, - {0xc45d1df942711d9a, 0x3ba5d0bd324f8395}, - {0xf5746577930d6500, 0xca8f44ec7ee3647a}, - {0x9968bf6abbe85f20, 0x7e998b13cf4e1ecc}, - {0xbfc2ef456ae276e8, 0x9e3fedd8c321a67f}, - {0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101f}, - {0x95d04aee3b80ece5, 0xbba1f1d158724a13}, - {0xbb445da9ca61281f, 0x2a8a6e45ae8edc98}, - {0xea1575143cf97226, 0xf52d09d71a3293be}, - {0x924d692ca61be758, 0x593c2626705f9c57}, - {0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836d}, - {0xe498f455c38b997a, 0x0b6dfb9c0f956448}, - {0x8edf98b59a373fec, 0x4724bd4189bd5ead}, - {0xb2977ee300c50fe7, 0x58edec91ec2cb658}, - {0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ee}, - {0x8b865b215899f46c, 0xbd79e0d20082ee75}, - {0xae67f1e9aec07187, 0xecd8590680a3aa12}, - {0xda01ee641a708de9, 0xe80e6f4820cc9496}, - {0x884134fe908658b2, 0x3109058d147fdcde}, - {0xaa51823e34a7eede, 0xbd4b46f0599fd416}, - {0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91b}, - {0x850fadc09923329e, 0x03e2cf6bc604ddb1}, - {0xa6539930bf6bff45, 0x84db8346b786151d}, - {0xcfe87f7cef46ff16, 0xe612641865679a64}, - {0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07f}, - {0xa26da3999aef7749, 0xe3be5e330f38f09e}, - {0xcb090c8001ab551c, 0x5cadf5bfd3072cc6}, - {0xfdcb4fa002162a63, 0x73d9732fc7c8f7f7}, - {0x9e9f11c4014dda7e, 0x2867e7fddcdd9afb}, - {0xc646d63501a1511d, 0xb281e1fd541501b9}, - {0xf7d88bc24209a565, 0x1f225a7ca91a4227}, - {0x9ae757596946075f, 0x3375788de9b06959}, - {0xc1a12d2fc3978937, 0x0052d6b1641c83af}, - {0xf209787bb47d6b84, 0xc0678c5dbd23a49b}, - {0x9745eb4d50ce6332, 0xf840b7ba963646e1}, - {0xbd176620a501fbff, 0xb650e5a93bc3d899}, - {0xec5d3fa8ce427aff, 0xa3e51f138ab4cebf}, - {0x93ba47c980e98cdf, 0xc66f336c36b10138}, - {0xb8a8d9bbe123f017, 0xb80b0047445d4185}, - {0xe6d3102ad96cec1d, 0xa60dc059157491e6}, - {0x9043ea1ac7e41392, 0x87c89837ad68db30}, - {0xb454e4a179dd1877, 0x29babe4598c311fc}, - {0xe16a1dc9d8545e94, 0xf4296dd6fef3d67b}, - {0x8ce2529e2734bb1d, 0x1899e4a65f58660d}, - {0xb01ae745b101e9e4, 0x5ec05dcff72e7f90}, - {0xdc21a1171d42645d, 0x76707543f4fa1f74}, - {0x899504ae72497eba, 0x6a06494a791c53a9}, - {0xabfa45da0edbde69, 0x0487db9d17636893}, - {0xd6f8d7509292d603, 0x45a9d2845d3c42b7}, - {0x865b86925b9bc5c2, 0x0b8a2392ba45a9b3}, - {0xa7f26836f282b732, 0x8e6cac7768d7141f}, - {0xd1ef0244af2364ff, 0x3207d795430cd927}, - {0x8335616aed761f1f, 0x7f44e6bd49e807b9}, - {0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a7}, - {0xcd036837130890a1, 0x36dba887c37a8c10}, - {0x802221226be55a64, 0xc2494954da2c978a}, - {0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6d}, - {0xc83553c5c8965d3d, 0x6f92829494e5acc8}, - {0xfa42a8b73abbf48c, 0xcb772339ba1f17fa}, - {0x9c69a97284b578d7, 0xff2a760414536efc}, - {0xc38413cf25e2d70d, 0xfef5138519684abb}, - {0xf46518c2ef5b8cd1, 0x7eb258665fc25d6a}, - {0x98bf2f79d5993802, 0xef2f773ffbd97a62}, - {0xbeeefb584aff8603, 0xaafb550ffacfd8fb}, - {0xeeaaba2e5dbf6784, 0x95ba2a53f983cf39}, - {0x952ab45cfa97a0b2, 0xdd945a747bf26184}, - {0xba756174393d88df, 0x94f971119aeef9e5}, - {0xe912b9d1478ceb17, 0x7a37cd5601aab85e}, - {0x91abb422ccb812ee, 0xac62e055c10ab33b}, - {0xb616a12b7fe617aa, 0x577b986b314d600a}, - {0xe39c49765fdf9d94, 0xed5a7e85fda0b80c}, - {0x8e41ade9fbebc27d, 0x14588f13be847308}, - {0xb1d219647ae6b31c, 0x596eb2d8ae258fc9}, - {0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bc}, - {0x8aec23d680043bee, 0x25de7bb9480d5855}, - {0xada72ccc20054ae9, 0xaf561aa79a10ae6b}, - {0xd910f7ff28069da4, 0x1b2ba1518094da05}, - {0x87aa9aff79042286, 0x90fb44d2f05d0843}, - {0xa99541bf57452b28, 0x353a1607ac744a54}, - {0xd3fa922f2d1675f2, 0x42889b8997915ce9}, - {0x847c9b5d7c2e09b7, 0x69956135febada12}, - {0xa59bc234db398c25, 0x43fab9837e699096}, - {0xcf02b2c21207ef2e, 0x94f967e45e03f4bc}, - {0x8161afb94b44f57d, 0x1d1be0eebac278f6}, - {0xa1ba1ba79e1632dc, 0x6462d92a69731733}, - {0xca28a291859bbf93, 0x7d7b8f7503cfdcff}, - {0xfcb2cb35e702af78, 0x5cda735244c3d43f}, - {0x9defbf01b061adab, 0x3a0888136afa64a8}, - {0xc56baec21c7a1916, 0x088aaa1845b8fdd1}, - {0xf6c69a72a3989f5b, 0x8aad549e57273d46}, - {0x9a3c2087a63f6399, 0x36ac54e2f678864c}, - {0xc0cb28a98fcf3c7f, 0x84576a1bb416a7de}, - {0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d6}, - {0x969eb7c47859e743, 0x9f644ae5a4b1b326}, - {0xbc4665b596706114, 0x873d5d9f0dde1fef}, - {0xeb57ff22fc0c7959, 0xa90cb506d155a7eb}, - {0x9316ff75dd87cbd8, 0x09a7f12442d588f3}, - {0xb7dcbf5354e9bece, 0x0c11ed6d538aeb30}, - {0xe5d3ef282a242e81, 0x8f1668c8a86da5fb}, - {0x8fa475791a569d10, 0xf96e017d694487bd}, - {0xb38d92d760ec4455, 0x37c981dcc395a9ad}, - {0xe070f78d3927556a, 0x85bbe253f47b1418}, - {0x8c469ab843b89562, 0x93956d7478ccec8f}, - {0xaf58416654a6babb, 0x387ac8d1970027b3}, - {0xdb2e51bfe9d0696a, 0x06997b05fcc0319f}, - {0x88fcf317f22241e2, 0x441fece3bdf81f04}, - {0xab3c2fddeeaad25a, 0xd527e81cad7626c4}, - {0xd60b3bd56a5586f1, 0x8a71e223d8d3b075}, - {0x85c7056562757456, 0xf6872d5667844e4a}, - {0xa738c6bebb12d16c, 0xb428f8ac016561dc}, - {0xd106f86e69d785c7, 0xe13336d701beba53}, - {0x82a45b450226b39c, 0xecc0024661173474}, - {0xa34d721642b06084, 0x27f002d7f95d0191}, - {0xcc20ce9bd35c78a5, 0x31ec038df7b441f5}, - {0xff290242c83396ce, 0x7e67047175a15272}, - {0x9f79a169bd203e41, 0x0f0062c6e984d387}, - {0xc75809c42c684dd1, 0x52c07b78a3e60869}, - {0xf92e0c3537826145, 0xa7709a56ccdf8a83}, - {0x9bbcc7a142b17ccb, 0x88a66076400bb692}, - {0xc2abf989935ddbfe, 0x6acff893d00ea436}, - {0xf356f7ebf83552fe, 0x0583f6b8c4124d44}, - {0x98165af37b2153de, 0xc3727a337a8b704b}, - {0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5d}, - {0xeda2ee1c7064130c, 0x1162def06f79df74}, - {0x9485d4d1c63e8be7, 0x8addcb5645ac2ba9}, - {0xb9a74a0637ce2ee1, 0x6d953e2bd7173693}, - {0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0438}, - {0x910ab1d4db9914a0, 0x1d9c9892400a22a3}, - {0xb54d5e4a127f59c8, 0x2503beb6d00cab4c}, - {0xe2a0b5dc971f303a, 0x2e44ae64840fd61e}, - {0x8da471a9de737e24, 0x5ceaecfed289e5d3}, - {0xb10d8e1456105dad, 0x7425a83e872c5f48}, - {0xdd50f1996b947518, 0xd12f124e28f7771a}, - {0x8a5296ffe33cc92f, 0x82bd6b70d99aaa70}, - {0xace73cbfdc0bfb7b, 0x636cc64d1001550c}, - {0xd8210befd30efa5a, 0x3c47f7e05401aa4f}, - {0x8714a775e3e95c78, 0x65acfaec34810a72}, - {0xa8d9d1535ce3b396, 0x7f1839a741a14d0e}, - {0xd31045a8341ca07c, 0x1ede48111209a051}, - {0x83ea2b892091e44d, 0x934aed0aab460433}, - {0xa4e4b66b68b65d60, 0xf81da84d56178540}, - {0xce1de40642e3f4b9, 0x36251260ab9d668f}, - {0x80d2ae83e9ce78f3, 0xc1d72b7c6b42601a}, - {0xa1075a24e4421730, 0xb24cf65b8612f820}, - {0xc94930ae1d529cfc, 0xdee033f26797b628}, - {0xfb9b7cd9a4a7443c, 0x169840ef017da3b2}, - {0x9d412e0806e88aa5, 0x8e1f289560ee864f}, - {0xc491798a08a2ad4e, 0xf1a6f2bab92a27e3}, - {0xf5b5d7ec8acb58a2, 0xae10af696774b1dc}, - {0x9991a6f3d6bf1765, 0xacca6da1e0a8ef2a}, - {0xbff610b0cc6edd3f, 0x17fd090a58d32af4}, - {0xeff394dcff8a948e, 0xddfc4b4cef07f5b1}, - {0x95f83d0a1fb69cd9, 0x4abdaf101564f98f}, - {0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f2}, - {0xea53df5fd18d5513, 0x84c86189216dc5ee}, - {0x92746b9be2f8552c, 0x32fd3cf5b4e49bb5}, - {0xb7118682dbb66a77, 0x3fbc8c33221dc2a2}, - {0xe4d5e82392a40515, 0x0fabaf3feaa5334b}, - {0x8f05b1163ba6832d, 0x29cb4d87f2a7400f}, - {0xb2c71d5bca9023f8, 0x743e20e9ef511013}, - {0xdf78e4b2bd342cf6, 0x914da9246b255417}, - {0x8bab8eefb6409c1a, 0x1ad089b6c2f7548f}, - {0xae9672aba3d0c320, 0xa184ac2473b529b2}, - {0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741f}, - {0x8865899617fb1871, 0x7e2fa67c7a658893}, - {0xaa7eebfb9df9de8d, 0xddbb901b98feeab8}, - {0xd51ea6fa85785631, 0x552a74227f3ea566}, - {0x8533285c936b35de, 0xd53a88958f872760}, - {0xa67ff273b8460356, 0x8a892abaf368f138}, - {0xd01fef10a657842c, 0x2d2b7569b0432d86}, - {0x8213f56a67f6b29b, 0x9c3b29620e29fc74}, - {0xa298f2c501f45f42, 0x8349f3ba91b47b90}, - {0xcb3f2f7642717713, 0x241c70a936219a74}, - {0xfe0efb53d30dd4d7, 0xed238cd383aa0111}, - {0x9ec95d1463e8a506, 0xf4363804324a40ab}, - {0xc67bb4597ce2ce48, 0xb143c6053edcd0d6}, - {0xf81aa16fdc1b81da, 0xdd94b7868e94050b}, - {0x9b10a4e5e9913128, 0xca7cf2b4191c8327}, - {0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f1}, - {0xf24a01a73cf2dccf, 0xbc633b39673c8ced}, - {0x976e41088617ca01, 0xd5be0503e085d814}, - {0xbd49d14aa79dbc82, 0x4b2d8644d8a74e19}, - {0xec9c459d51852ba2, 0xddf8e7d60ed1219f}, - {0x93e1ab8252f33b45, 0xcabb90e5c942b504}, - {0xb8da1662e7b00a17, 0x3d6a751f3b936244}, - {0xe7109bfba19c0c9d, 0x0cc512670a783ad5}, - {0x906a617d450187e2, 0x27fb2b80668b24c6}, - {0xb484f9dc9641e9da, 0xb1f9f660802dedf7}, - {0xe1a63853bbd26451, 0x5e7873f8a0396974}, - {0x8d07e33455637eb2, 0xdb0b487b6423e1e9}, - {0xb049dc016abc5e5f, 0x91ce1a9a3d2cda63}, - {0xdc5c5301c56b75f7, 0x7641a140cc7810fc}, - {0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9e}, - {0xac2820d9623bf429, 0x546345fa9fbdcd45}, - {0xd732290fbacaf133, 0xa97c177947ad4096}, - {0x867f59a9d4bed6c0, 0x49ed8eabcccc485e}, - {0xa81f301449ee8c70, 0x5c68f256bfff5a75}, - {0xd226fc195c6a2f8c, 0x73832eec6fff3112}, - {0x83585d8fd9c25db7, 0xc831fd53c5ff7eac}, - {0xa42e74f3d032f525, 0xba3e7ca8b77f5e56}, - {0xcd3a1230c43fb26f, 0x28ce1bd2e55f35ec}, - {0x80444b5e7aa7cf85, 0x7980d163cf5b81b4}, - {0xa0555e361951c366, 0xd7e105bcc3326220}, - {0xc86ab5c39fa63440, 0x8dd9472bf3fefaa8}, - {0xfa856334878fc150, 0xb14f98f6f0feb952}, - {0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d4}, - {0xc3b8358109e84f07, 0x0a862f80ec4700c9}, - {0xf4a642e14c6262c8, 0xcd27bb612758c0fb}, - {0x98e7e9cccfbd7dbd, 0x8038d51cb897789d}, - {0xbf21e44003acdd2c, 0xe0470a63e6bd56c4}, - {0xeeea5d5004981478, 0x1858ccfce06cac75}, - {0x95527a5202df0ccb, 0x0f37801e0c43ebc9}, - {0xbaa718e68396cffd, 0xd30560258f54e6bb}, - {0xe950df20247c83fd, 0x47c6b82ef32a206a}, - {0x91d28b7416cdd27e, 0x4cdc331d57fa5442}, - {0xb6472e511c81471d, 0xe0133fe4adf8e953}, - {0xe3d8f9e563a198e5, 0x58180fddd97723a7}, - {0x8e679c2f5e44ff8f, 0x570f09eaa7ea7649}, - {0xb201833b35d63f73, 0x2cd2cc6551e513db}, - {0xde81e40a034bcf4f, 0xf8077f7ea65e58d2}, - {0x8b112e86420f6191, 0xfb04afaf27faf783}, - {0xadd57a27d29339f6, 0x79c5db9af1f9b564}, - {0xd94ad8b1c7380874, 0x18375281ae7822bd}, - {0x87cec76f1c830548, 0x8f2293910d0b15b6}, - {0xa9c2794ae3a3c69a, 0xb2eb3875504ddb23}, - {0xd433179d9c8cb841, 0x5fa60692a46151ec}, - {0x849feec281d7f328, 0xdbc7c41ba6bcd334}, - {0xa5c7ea73224deff3, 0x12b9b522906c0801}, - {0xcf39e50feae16bef, 0xd768226b34870a01}, - {0x81842f29f2cce375, 0xe6a1158300d46641}, - {0xa1e53af46f801c53, 0x60495ae3c1097fd1}, - {0xca5e89b18b602368, 0x385bb19cb14bdfc5}, - {0xfcf62c1dee382c42, 0x46729e03dd9ed7b6}, - {0x9e19db92b4e31ba9, 0x6c07a2c26a8346d2}, - {0xc5a05277621be293, 0xc7098b7305241886}, - {0xf70867153aa2db38, 0xb8cbee4fc66d1ea8} -}; - -// Per-format helper routines (binary64 specializations of the Dragonbox steps). -struct compute_mul_result { - std::uint64_t integer_part; - bool is_integer; -}; -struct compute_mul_parity_result { - bool parity; - bool is_integer; -}; - -inline compute_mul_result compute_mul(std::uint64_t u, uint128 c) noexcept { - const uint128 r = umul192_upper128(u, c); - return {r.high, r.low == 0}; -} - -inline std::uint64_t compute_delta(uint128 c, int beta) noexcept { - return c.high >> (total_bits - 1 - beta); -} - -inline compute_mul_parity_result compute_mul_parity(std::uint64_t two_f, - uint128 c, int beta) noexcept { - // beta is always in [1, 63] here. - const uint128 r = umul192_lower128(two_f, c); - return {((r.high >> (64 - beta)) & 1) != 0, - ((r.high << beta) | (r.low >> (64 - beta))) == 0}; -} - -inline std::uint64_t -compute_left_endpoint_for_shorter_interval_case(uint128 c, int beta) noexcept { - return (c.high - (c.high >> (significand_bits + 2))) >> - (total_bits - significand_bits - 1 - beta); -} - -inline std::uint64_t -compute_right_endpoint_for_shorter_interval_case(uint128 c, int beta) noexcept { - return (c.high + (c.high >> (significand_bits + 1))) >> - (total_bits - significand_bits - 1 - beta); -} - -inline std::uint64_t -compute_round_up_for_shorter_interval_case(uint128 c, int beta) noexcept { - return ((c.high >> (total_bits - significand_bits - 2 - beta)) + 1) / 2; -} - -// floor(n / 10) for the shorter-interval right endpoint (n bounded so the -// single multiply below is exact). -inline std::uint64_t divide_by_pow10_1(std::uint64_t n) noexcept { - return umul128_upper64(n, std::uint64_t(1844674407370955162ull)); -} - -// floor(n / 1000) for the larger-divisor step (n bounded as above). -inline std::uint64_t divide_by_pow10_3(std::uint64_t n) noexcept { - return umul128_upper64(n, std::uint64_t(4722366482869645214ull)) >> 8; -} - -// Returns whether n is divisible by 10^kappa (= 100) and divides n by it. -inline bool check_divisibility_and_divide_by_pow10_kappa(std::uint64_t &n) noexcept { - // magic number for division by 100 (kappa == 2). - const std::uint32_t prod = std::uint32_t(n) * std::uint32_t(656); - const bool result = (prod & 0xffffu) < 656u; - n = std::uint64_t(prod >> 16); - return result; -} - -// Strip trailing decimal zeros from significand, bumping exponent accordingly. -// Branchless search; constants from the Dragonbox reference. -inline void remove_trailing_zeros(std::uint64_t &significand, int &exponent) noexcept { - std::uint64_t r = rotr64(significand * std::uint64_t(28999941890838049ull), 8); - bool b = r < std::uint64_t(184467440738ull); - int s = b ? 1 : 0; - significand = b ? r : significand; - - r = rotr64(significand * std::uint64_t(182622766329724561ull), 4); - b = r < std::uint64_t(1844674407370956ull); - s = s * 2 + (b ? 1 : 0); - significand = b ? r : significand; - - r = rotr64(significand * std::uint64_t(10330176681277348905ull), 2); - b = r < std::uint64_t(184467440737095517ull); - s = s * 2 + (b ? 1 : 0); - significand = b ? r : significand; - - r = rotr64(significand * std::uint64_t(14757395258967641293ull), 1); - b = r < std::uint64_t(1844674407370955162ull); - s = s * 2 + (b ? 1 : 0); - significand = b ? r : significand; - - exponent += s; -} - -// Dragonbox core: shortest (significand, exponent) such that -// value == significand * 10^exponent -// for a finite, positive, non-zero binary64 value, decomposed into its raw -// significand bits and biased exponent bits. -struct decimal_fp { - std::uint64_t significand; - int exponent; -}; - -inline decimal_fp to_decimal(std::uint64_t binary_significand, - int binary_exponent) noexcept { - const bool is_even = (binary_significand % 2 == 0); - std::uint64_t two_fc = binary_significand * 2; - - // Is the input a normal number? - if (binary_exponent != 0) { - binary_exponent += exponent_bias - significand_bits; - - // Shorter interval case; proceed like Schubfach. - if (two_fc == 0) { - const int minus_k = - floor_log10_pow2_minus_log10_4_over_3(binary_exponent); - const int beta = binary_exponent + floor_log2_pow10(-minus_k); - const uint128 c = cache[-minus_k - cache_min_k]; - - std::uint64_t xi = - compute_left_endpoint_for_shorter_interval_case(c, beta); - const std::uint64_t zi = - compute_right_endpoint_for_shorter_interval_case(c, beta); - - // If the left endpoint is not an integer, increase it. (Both endpoints - // are always included since the significand is even.) - if (!(binary_exponent >= - case_shorter_interval_left_endpoint_lower_threshold && - binary_exponent <= - case_shorter_interval_left_endpoint_upper_threshold)) { - ++xi; - } - - // Try the bigger divisor. - std::uint64_t decimal_significand = divide_by_pow10_1(zi); - if (decimal_significand * 10 >= xi) { - int decimal_exponent = minus_k + 1; - remove_trailing_zeros(decimal_significand, decimal_exponent); - return {decimal_significand, decimal_exponent}; - } - - // Otherwise, compute the round-up of y. - decimal_significand = - compute_round_up_for_shorter_interval_case(c, beta); - // On a tie, choose the even one. - if ((decimal_significand % 2 != 0) && - binary_exponent >= shorter_interval_tie_lower_threshold && - binary_exponent <= shorter_interval_tie_upper_threshold) { - --decimal_significand; - } else if (decimal_significand < xi) { - ++decimal_significand; - } - return {decimal_significand, minus_k}; - } - - // Normal interval case. - two_fc |= (std::uint64_t(1) << (significand_bits + 1)); - } else { - // Subnormal number: normal interval case. - binary_exponent = min_exponent - significand_bits; - } - - // Step 1: Schubfach multiplier calculation. - const int minus_k = floor_log10_pow2(binary_exponent) - kappa; - const uint128 c = cache[-minus_k - cache_min_k]; - const int beta = binary_exponent + floor_log2_pow10(-minus_k); - - const std::uint64_t deltai = compute_delta(c, beta); - const compute_mul_result z_result = - compute_mul((two_fc | 1) << beta, c); - - // Step 2: Try larger divisor; remove trailing zeros if necessary. - std::uint64_t decimal_significand = divide_by_pow10_3(z_result.integer_part); - std::uint64_t r = - z_result.integer_part - std::uint64_t(big_divisor) * decimal_significand; - - do { - if (r < deltai) { - // Exclude the right endpoint if necessary. - if ((r | std::uint64_t(!z_result.is_integer) | std::uint64_t(is_even)) == - 0) { - --decimal_significand; - r = big_divisor; - break; - } - } else if (r > deltai) { - break; - } else { - // r == deltai; compare fractional parts. - const compute_mul_parity_result x_result = - compute_mul_parity(two_fc - 1, c, beta); - if (!(x_result.parity | (x_result.is_integer & is_even))) { - break; - } - } - - int decimal_exponent = minus_k + kappa + 1; - remove_trailing_zeros(decimal_significand, decimal_exponent); - return {decimal_significand, decimal_exponent}; - } while (false); - - // Step 3: Find the significand with the smaller divisor. - decimal_significand *= 10; - - std::uint64_t dist = r - (deltai / 2) + (small_divisor / 2); - const bool approx_y_parity = ((dist ^ (small_divisor / 2)) & 1) != 0; - - const bool divisible_by_small_divisor = - check_divisibility_and_divide_by_pow10_kappa(dist); - - decimal_significand += dist; - - if (divisible_by_small_divisor) { - const compute_mul_parity_result y_result = - compute_mul_parity(two_fc, c, beta); - if (y_result.parity != approx_y_parity) { - --decimal_significand; - } else if ((decimal_significand % 2) != 0 && y_result.is_integer) { - // On a tie (y is an integer), choose the even one. - --decimal_significand; - } - } - - return {decimal_significand, minus_k + kappa}; -} - -/*! -Fills buf with the shortest decimal digits of 'value' (which must be finite, -positive and non-zero), sets len to the number of digits, and decimal_exponent -so that value == (buf, interpreted as an integer) * 10^decimal_exponent. -This mirrors the contract of the previous grisu2() entry point. -*/ -inline void dragonbox(char *buf, int &len, int &decimal_exponent, - double value) { - std::uint64_t bits; - std::memcpy(&bits, &value, sizeof(bits)); - const std::uint64_t binary_significand = - bits & ((std::uint64_t(1) << significand_bits) - 1); - const int binary_exponent = - int((bits >> significand_bits) & 0x7ff); - - const decimal_fp dec = to_decimal(binary_significand, binary_exponent); - - // Convert the decimal significand to digits: - // 1) Proceed 2 digits at a time (s % 100) via a 00..99 lookup table - // (see Alexandrescu, "Three Optimization Tips for C++", 2012), - // 2) Digits come out least-significant first, writing them back-to-front - // with p = tmp + sizeof(tmp); to avoid reversal pass - // 3) Proceed remaining digits after loop to avoid branchs inside it - // 4) memcpy digits to char* buf (inside function input) - static const char digits2[201] = - "0001020304050607080910111213141516171819" - "2021222324252627282930313233343536373839" - "4041424344454647484950515253545556575859" - "6061626364656667686970717273747576777879" - "8081828384858687888990919293949596979899"; - // Digit area is 24; +16 padding lets us always memcpy 16 (+1) bytes with a - // compile-time size so the compiler inlines (no libc size-class branches). - // Callers must provide to_chars_buffer_size (40) bytes for the same reason. - constexpr int digit_area = 24; - char tmp[digit_area + 16]; - char *p = tmp + digit_area; // write backward - std::uint64_t s = dec.significand; - while (s >= 100) { - const std::uint32_t idx = static_cast(s % 100) * 2; - s /= 100; - p -= 2; - p[0] = digits2[idx]; - p[1] = digits2[idx + 1]; - } - if (s >= 10) { - const std::uint32_t idx = static_cast(s) * 2; - p -= 2; - p[0] = digits2[idx]; - p[1] = digits2[idx + 1]; - } else { - *--p = static_cast('0' + s); - } - // Fixed-size copy: double has at most 17 significant digits. - std::memcpy(buf, p, 16); - buf[16] = p[16]; - len = static_cast(tmp + digit_area - p); - decimal_exponent = dec.exponent; -} - -/*! -@brief appends a decimal representation of e to buf -@return a pointer to the element following the exponent. -@pre -1000 < e < 1000 -*/ -inline char *append_exponent(char *buf, int e) { - - bool isNegative = e < 0; - e = isNegative ? -e : e; - *buf++ = isNegative ? '-' : '+'; - - auto k = static_cast(e); - if (k < 100) { - // Always print at least two digits in the exponent. - // This is for compatibility with printf("%g"). - *buf++ = static_cast('0' + k / 10); - k %= 10; - *buf++ = static_cast('0' + k); - } else { - *buf++ = static_cast('0' + k / 100); - k %= 100; - *buf++ = static_cast('0' + k / 10); - k %= 10; - *buf++ = static_cast('0' + k); - } - - return buf; -} - -/*! -@brief prettify v = buf * 10^decimal_exponent -If v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point -notation. Otherwise it will be printed in exponential notation. -@pre min_exp < 0 -@pre max_exp > 0 -*/ -inline char *format_buffer(char *buf, int len, int decimal_exponent, - int min_exp, int max_exp) { - - const int k = len; - const int n = len + decimal_exponent; - - // v = buf * 10^(n-k) - // k is the length of the buffer (number of decimal digits) - // n is the position of the decimal point relative to the start of the buffer. - - // All mem* sizes below are compile-time constants so the compiler inlines - // them as plain loads/stores. That requires over-writing past the logical - // string length; callers must reserve to_chars_buffer_size (40) bytes. - // Logical output is still bounded by ~24 characters; only the returned - // pointer reflects the true length. - - if (k <= n && n <= max_exp) { - // digits[000] - // len <= max_exp + 2 - std::memset(buf + k, '0', 16); - // Make it look like a floating-point number (#362, #378) - buf[n + 0] = '.'; - buf[n + 1] = '0'; - return buf + (static_cast(n)) + 2; - } - - if (0 < n && n <= max_exp) { - // dig.its - // len <= max_digits10 + 1 - // Shift the fractional digits one place right via a temp (overlap). - char shifted[16]; - std::memcpy(shifted, buf + static_cast(n), 16); - std::memcpy(buf + (static_cast(n) + 1), shifted, 16); - buf[n] = '.'; - return buf + (static_cast(k) + 1U); - } - - if (min_exp < n && n <= 0) { - // 0.[000]digits - // With kMinExp = -4, n is in {-3,-2,-1,0}, so pad = -n is 0..3. - // len <= 2 + (-min_exp - 1) + max_digits10 - char digits[17]; - std::memcpy(digits, buf, 17); - const size_t pad = static_cast(-n); // 0..3 - buf[0] = '0'; - buf[1] = '.'; - // Fixed upper bound on leading zeros; only the first `pad` matter. - std::memset(buf + 2, '0', 4); - std::memcpy(buf + 2 + pad, digits, 17); - return buf + (2U + pad + static_cast(k)); - } - - if (k == 1) { - // dE+123 - // len <= 1 + 5 - buf += 1; - } else { - // d.igitsE+123 - // len <= max_digits10 + 1 + 5 - // k-1 <= 16 for double; fixed-size shift via temp (overlap). - char shifted[16]; - std::memcpy(shifted, buf + 1, 16); - std::memcpy(buf + 2, shifted, 16); - buf[1] = '.'; - buf += 1 + static_cast(k); - } - - *buf++ = 'e'; - return append_exponent(buf, n - 1); -} - -} // NS dtoa_impl - -/*! -The format of the resulting decimal representation is similar to printf's %g -format. Returns an iterator pointing past-the-end of the decimal representation. -@note The input number must be finite, i.e. NaN's and Inf's are not supported. -@note The buffer must have at least to_chars_buffer_size (40) writable bytes. - Only ~24 characters are ever part of the logical result, but fixed-size - 16/17-byte mem* over-writes require the extra scratch for safety. -@note The result is NOT null-terminated. -*/ -char *to_chars(char *first, const char *last, double value) { - static_cast(last); // maybe unused - fix warning - bool negative = std::signbit(value); - if (negative) { - value = -value; - *first++ = '-'; - } - - if (value == 0) // +-0 - { - *first++ = '0'; - // Make it look like a floating-point number (#362, #378) - *first++ = '.'; - *first++ = '0'; - return first; - } - // Compute v = buffer * 10^decimal_exponent. - // The decimal digits are stored in the buffer, which needs to be interpreted - // as an unsigned decimal integer. - // len is the length of the buffer, i.e. the number of decimal digits. - int len = 0; - int decimal_exponent = 0; - dtoa_impl::dragonbox(first, len, decimal_exponent, value); - // Format the buffer like printf("%.*g", prec, value) - constexpr int kMinExp = -4; - constexpr int kMaxExp = std::numeric_limits::digits10; - - return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, - kMaxExp); -} -} // NS internal -} // NS simdjson - -#endif // SIMDJSON_SRC_TO_CHARS_CPP - - -/* end file to_chars.cpp */ -/* including from_chars.cpp: #include */ -/* begin file from_chars.cpp */ -#ifndef SIMDJSON_SRC_FROM_CHARS_CPP -#define SIMDJSON_SRC_FROM_CHARS_CPP - -/* skipped duplicate #include */ - -/* including simdjson/internal/fast_float.h: #include "simdjson/internal/fast_float.h" */ -/* begin file simdjson/internal/fast_float.h */ -// Vendored from fast_float v8.2.10, generated by tools/vendor_fast_float.sh. -// Do not edit by hand; re-run the script to update. -// -// https://github.com/fastfloat/fast_float -// Licensed under Apache-2.0 OR MIT OR BSL-1.0, at your option. -// -// simdjson uses this for two things that its own number parser cannot do: -// * the slow path for numbers with more than 19 significant digits, where -// fast_float's bigint comparison is several times quicker than the -// Wuffs-derived decimal shifting it replaced (see src/from_chars.cpp), and -// * correctly rounded parsing inside a constant expression, which the runtime -// path cannot offer because it relies on memcpy and __uint128_t (see -// compile_time_json-inl.h). -// -// Two edits are applied by the script. Every fast_float name is rewritten so -// that this copy cannot collide with a copy of fast_float that the surrounding -// program includes for itself: namespace fast_float -> simdjson_fast_float, -// FASTFLOAT_* -> SIMDJSON_FASTFLOAT_*, fastfloat_* -> simdjson_fastfloat_*. And -// the accented letters in the attribution comments below are folded to ASCII, -// to keep the tree ASCII-only; no disrespect to the people named is intended. -// simdjson_fast_float by Daniel Lemire -// simdjson_fast_float by Joao Paulo Magalhaes -// -// -// with contributions from Eugene Golushkov -// with contributions from Maksim Kita -// with contributions from Marcin Wojdyr -// with contributions from Neal Richardson -// with contributions from Tim Paine -// with contributions from Fabio Pellacini -// with contributions from Lenard Szolnoki -// with contributions from Jan Pharago -// with contributions from Maya Warrier -// with contributions from Taha Khokhar -// with contributions from Anders Dalvander -// -// -// Licensed under the Apache License, Version 2.0, or the -// MIT License or the Boost License. This file may not be copied, -// modified, or distributed except according to those terms. -// -// MIT License Notice -// -// MIT License -// -// Copyright (c) 2021 The simdjson_fast_float authors -// -// Permission is hereby granted, free of charge, to any -// person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the -// Software without restriction, including without -// limitation the rights to use, copy, modify, merge, -// publish, distribute, sublicense, and/or sell copies of -// the Software, and to permit persons to whom the Software -// is furnished to do so, subject to the following -// conditions: -// -// The above copyright notice and this permission notice -// shall be included in all copies or substantial portions -// of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -// -// Apache License (Version 2.0) Notice -// -// Copyright 2021 The simdjson_fast_float authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// -// BOOST License Notice -// -// Boost Software License - Version 1.0 - August 17th, 2003 -// -// Permission is hereby granted, free of charge, to any person or organization -// obtaining a copy of the software and accompanying documentation covered by -// this license (the "Software") to use, reproduce, display, distribute, -// execute, and transmit the Software, and to prepare derivative works of the -// Software, and to permit third-parties to whom the Software is furnished to -// do so, all subject to the following: -// -// The copyright notices in the Software and this entire statement, including -// the above license grant, this restriction and the following disclaimer, -// must be included in all copies of the Software, in whole or in part, and -// all derivative works of the Software, unless such copies or derivative -// works are solely in the form of machine-executable object code generated by -// a source language processor. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -// - -#ifndef SIMDJSON_FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H -#define SIMDJSON_FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H - -#ifdef __has_include -#if __has_include() -#include -#endif -#endif - -// Testing for https://wg21.link/N3652, adopted in C++14 -#if defined(__cpp_constexpr) && __cpp_constexpr >= 201304 -#define SIMDJSON_FASTFLOAT_CONSTEXPR14 constexpr -#else -#define SIMDJSON_FASTFLOAT_CONSTEXPR14 -#endif - -#if defined(__cpp_lib_bit_cast) && __cpp_lib_bit_cast >= 201806L -#define SIMDJSON_FASTFLOAT_HAS_BIT_CAST 1 -#else -#define SIMDJSON_FASTFLOAT_HAS_BIT_CAST 0 -#endif - -#if defined(__cpp_lib_is_constant_evaluated) && \ - __cpp_lib_is_constant_evaluated >= 201811L -#define SIMDJSON_FASTFLOAT_HAS_IS_CONSTANT_EVALUATED 1 -#else -#define SIMDJSON_FASTFLOAT_HAS_IS_CONSTANT_EVALUATED 0 -#endif - -#if defined(__cpp_if_constexpr) && __cpp_if_constexpr >= 201606L -#define SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(x) if constexpr (x) -#else -#define SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(x) if (x) -#endif - -// Testing for relevant C++20 constexpr library features -#if SIMDJSON_FASTFLOAT_HAS_IS_CONSTANT_EVALUATED && SIMDJSON_FASTFLOAT_HAS_BIT_CAST && \ - defined(__cpp_lib_constexpr_algorithms) && \ - __cpp_lib_constexpr_algorithms >= 201806L /*For std::copy and std::fill*/ -#define SIMDJSON_FASTFLOAT_CONSTEXPR20 constexpr -#define SIMDJSON_FASTFLOAT_IS_CONSTEXPR 1 -#else -#define SIMDJSON_FASTFLOAT_CONSTEXPR20 -#define SIMDJSON_FASTFLOAT_IS_CONSTEXPR 0 -#endif - -#if __cplusplus >= 201703L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) -#define SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE 0 -#else -#define SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE 1 -#endif - -#endif // SIMDJSON_FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H - -#ifndef SIMDJSON_FASTFLOAT_FLOAT_COMMON_H -#define SIMDJSON_FASTFLOAT_FLOAT_COMMON_H - -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef __has_include -#if __has_include() && (__cplusplus > 202002L || (defined(_MSVC_LANG) && (_MSVC_LANG > 202002L))) -#include -#endif -#endif - -#define SIMDJSON_FASTFLOAT_VERSION_MAJOR 8 -#define SIMDJSON_FASTFLOAT_VERSION_MINOR 2 -#define SIMDJSON_FASTFLOAT_VERSION_PATCH 10 - -#define SIMDJSON_FASTFLOAT_STRINGIZE_IMPL(x) #x -#define SIMDJSON_FASTFLOAT_STRINGIZE(x) SIMDJSON_FASTFLOAT_STRINGIZE_IMPL(x) - -#define SIMDJSON_FASTFLOAT_VERSION_STR \ - SIMDJSON_FASTFLOAT_STRINGIZE(SIMDJSON_FASTFLOAT_VERSION_MAJOR) \ - "." SIMDJSON_FASTFLOAT_STRINGIZE(SIMDJSON_FASTFLOAT_VERSION_MINOR) "." SIMDJSON_FASTFLOAT_STRINGIZE( \ - SIMDJSON_FASTFLOAT_VERSION_PATCH) - -#define SIMDJSON_FASTFLOAT_VERSION \ - (SIMDJSON_FASTFLOAT_VERSION_MAJOR * 10000 + SIMDJSON_FASTFLOAT_VERSION_MINOR * 100 + \ - SIMDJSON_FASTFLOAT_VERSION_PATCH) - -namespace simdjson_fast_float { - -enum class chars_format : uint64_t; - -namespace detail { -constexpr chars_format basic_json_fmt = chars_format(1 << 5); -constexpr chars_format basic_fortran_fmt = chars_format(1 << 6); -} // namespace detail - -enum class chars_format : uint64_t { - scientific = 1 << 0, - fixed = 1 << 2, - hex = 1 << 3, - no_infnan = 1 << 4, - // RFC 8259: https://datatracker.ietf.org/doc/html/rfc8259#section-6 - json = uint64_t(detail::basic_json_fmt) | fixed | scientific | no_infnan, - // Extension of RFC 8259 where, e.g., "inf" and "nan" are allowed. - json_or_infnan = uint64_t(detail::basic_json_fmt) | fixed | scientific, - fortran = uint64_t(detail::basic_fortran_fmt) | fixed | scientific, - general = fixed | scientific, - allow_leading_plus = 1 << 7, - skip_white_space = 1 << 8, -}; - -template struct from_chars_result_t { - UC const *ptr; - std::errc ec; - - // https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/p2497r0.html - constexpr explicit operator bool() const noexcept { - return ec == std::errc(); - } -}; - -using from_chars_result = from_chars_result_t; - -template struct parse_options_t { - constexpr explicit parse_options_t(chars_format fmt = chars_format::general, - UC dot = UC('.'), int b = 10) - : format(fmt), decimal_point(dot), base(b) {} - - /** Which number formats are accepted */ - chars_format format; - /** The character used as decimal point */ - UC decimal_point; - /** The base used for integers */ - int base; -}; - -using parse_options = parse_options_t; - -} // namespace simdjson_fast_float - -#if SIMDJSON_FASTFLOAT_HAS_BIT_CAST -#include -#endif - -#if (defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ - defined(__amd64) || defined(__aarch64__) || defined(_M_ARM64) || \ - defined(__MINGW64__) || defined(__s390x__) || \ - (defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \ - defined(__PPC64LE__)) || \ - defined(__loongarch64) || (defined(__riscv) && __riscv_xlen == 64)) -#define SIMDJSON_FASTFLOAT_64BIT 1 -#elif (defined(__i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(__arm__) || defined(_M_ARM) || defined(__ppc__) || \ - defined(__MINGW32__) || defined(__EMSCRIPTEN__) || \ - (defined(__riscv) && __riscv_xlen == 32)) -#define SIMDJSON_FASTFLOAT_32BIT 1 -#else - // Need to check incrementally, since SIZE_MAX is a size_t, avoid overflow. -// We can never tell the register width, but the SIZE_MAX is a good -// approximation. UINTPTR_MAX and INTPTR_MAX are optional, so avoid them for max -// portability. -#if SIZE_MAX == 0xffff -#error Unknown platform (16-bit, unsupported) -#elif SIZE_MAX == 0xffffffff -#define SIMDJSON_FASTFLOAT_32BIT 1 -#elif SIZE_MAX == 0xffffffffffffffff -#define SIMDJSON_FASTFLOAT_64BIT 1 -#else -#error Unknown platform (not 32-bit, not 64-bit?) -#endif -#endif - -#if ((defined(_WIN32) || defined(_WIN64)) && !defined(__clang__)) || \ - (defined(_M_ARM64) && !defined(__MINGW32__)) -#include -#endif - -#if defined(_MSC_VER) && !defined(__clang__) -#define SIMDJSON_FASTFLOAT_VISUAL_STUDIO 1 -#endif - -#if defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__ -#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) -#elif defined _WIN32 -#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN 0 -#else -#if defined(__APPLE__) || defined(__FreeBSD__) -#include -#elif defined(sun) || defined(__sun) -#include -#elif defined(__MVS__) -#include -#else -#ifdef __has_include -#if __has_include() -#include -#endif //__has_include() -#endif //__has_include -#endif -# -#ifndef __BYTE_ORDER__ -// safe choice -#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN 0 -#endif -# -#ifndef __ORDER_LITTLE_ENDIAN__ -// safe choice -#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN 0 -#endif -# -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN 0 -#else -#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN 1 -#endif -#endif - -#if defined(__SSE2__) || (defined(SIMDJSON_FASTFLOAT_VISUAL_STUDIO) && \ - (defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && _M_IX86_FP == 2))) -#define SIMDJSON_FASTFLOAT_SSE2 1 -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) -#define SIMDJSON_FASTFLOAT_NEON 1 -#endif - -#if defined(SIMDJSON_FASTFLOAT_SSE2) || defined(SIMDJSON_FASTFLOAT_NEON) -#define SIMDJSON_FASTFLOAT_HAS_SIMD 1 -#endif - -#if defined(__GNUC__) -// disable -Wcast-align=strict (GCC only) -#define SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wcast-align\"") -#else -#define SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS -#endif - -#if defined(__GNUC__) -#define SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS _Pragma("GCC diagnostic pop") -#else -#define SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS -#endif - -#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO -#define simdjson_fastfloat_really_inline __forceinline -#else -#define simdjson_fastfloat_really_inline inline __attribute__((always_inline)) -#endif - -// Branch-probability hint marking the rare slow-path branches as cold, so the -// optimizer keeps the out-of-line slow-path re-parse off the hot path (and does -// not duplicate the force-inlined hot scanner into the caller, which bloated -// the hot frame and hurt ILP on some targets). Used at the call site as -// if simdjson_fastfloat_unlikely(cond) { ... } -// (the macro supplies the parentheses). It expands to the standard [[unlikely]] -// attribute when supported, otherwise to __builtin_expect on GCC/Clang, or -// to a no-op elsewhere (e.g. pre-C++20 MSVC, which has no equivalent hint). -#ifdef __has_cpp_attribute -#if __has_cpp_attribute(unlikely) >= 201803L -// g++-9 hits hits this branch, but then fails to compile -// [[unlikely]]. This happens only with g++-9. -#if !defined(__GNUC__) || (__GNUC__ != 9) -#define SIMDJSON_FASTFLOAT_USE_UNLIKELY_ATTR -#endif -#endif -#endif - -#ifdef SIMDJSON_FASTFLOAT_USE_UNLIKELY_ATTR -#define simdjson_fastfloat_unlikely(x) (x) [[unlikely]] -#elif defined(__GNUC__) || defined(__clang__) -#define simdjson_fastfloat_unlikely(x) (__builtin_expect(!!(x), 0)) -#else -#define simdjson_fastfloat_unlikely(x) (x) -#endif - -#ifndef SIMDJSON_FASTFLOAT_ASSERT -#define SIMDJSON_FASTFLOAT_ASSERT(x) \ - { \ - static_cast(x); \ - } -#endif - -#ifndef SIMDJSON_FASTFLOAT_DEBUG_ASSERT -#define SIMDJSON_FASTFLOAT_DEBUG_ASSERT(x) \ - { \ - static_cast(x); \ - } -#endif - -// rust style `try!()` macro, or `?` operator -#define SIMDJSON_FASTFLOAT_TRY(x) \ - { \ - if (!(x)) \ - return false; \ - } - -#define SIMDJSON_FASTFLOAT_ENABLE_IF(...) \ - typename std::enable_if<(__VA_ARGS__), int>::type - -namespace simdjson_fast_float { - -simdjson_fastfloat_really_inline constexpr bool cpp20_and_in_constexpr() { -#if SIMDJSON_FASTFLOAT_HAS_IS_CONSTANT_EVALUATED - return std::is_constant_evaluated(); -#else - return false; -#endif -} - -template -struct is_supported_float_type - : std::integral_constant< - bool, std::is_same::value || std::is_same::value -#ifdef __STDCPP_FLOAT64_T__ - || std::is_same::value -#endif -#ifdef __STDCPP_FLOAT32_T__ - || std::is_same::value -#endif -#ifdef __STDCPP_FLOAT16_T__ - || std::is_same::value -#endif -#ifdef __STDCPP_BFLOAT16_T__ - || std::is_same::value -#endif - > { -}; - -template -using equiv_uint_t = typename std::conditional< - sizeof(T) == 1, uint8_t, - typename std::conditional< - sizeof(T) == 2, uint16_t, - typename std::conditional::type>::type>::type; - -template struct is_supported_integer_type : std::is_integral {}; - -template -struct is_supported_char_type - : std::integral_constant::value || - std::is_same::value || - std::is_same::value || - std::is_same::value -#ifdef __cpp_char8_t - || std::is_same::value -#endif - > { -}; - -template -inline SIMDJSON_FASTFLOAT_CONSTEXPR14 bool -simdjson_fastfloat_strncasecmp3(UC const *actual_mixedcase, - UC const *expected_lowercase) { - uint64_t mask{0}; - SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1) { mask = 0x2020202020202020; } - else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 2) { - mask = 0x0020002000200020; - } - else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) { - mask = 0x0000002000000020; - } - else { - return false; - } - - uint64_t val1{0}, val2{0}; - if (cpp20_and_in_constexpr()) { - for (size_t i = 0; i < 3; i++) { - if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) { - return false; - } - } - return true; - } else { - SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1 || sizeof(UC) == 2) { - ::memcpy(&val1, actual_mixedcase, 3 * sizeof(UC)); - ::memcpy(&val2, expected_lowercase, 3 * sizeof(UC)); - val1 |= mask; - val2 |= mask; - return val1 == val2; - } - else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) { - ::memcpy(&val1, actual_mixedcase, 2 * sizeof(UC)); - ::memcpy(&val2, expected_lowercase, 2 * sizeof(UC)); - val1 |= mask; - if (val1 != val2) { - return false; - } - return (actual_mixedcase[2] | 32) == (expected_lowercase[2]); - } - else { - return false; - } - } -} - -template -inline SIMDJSON_FASTFLOAT_CONSTEXPR14 bool -simdjson_fastfloat_strncasecmp5(UC const *actual_mixedcase, - UC const *expected_lowercase) { - uint64_t mask{0}; - uint64_t val1{0}, val2{0}; - if (cpp20_and_in_constexpr()) { - for (size_t i = 0; i < 5; i++) { - if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) { - return false; - } - } - return true; - } else { - SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1) { - mask = 0x2020202020202020; - ::memcpy(&val1, actual_mixedcase, 5 * sizeof(UC)); - ::memcpy(&val2, expected_lowercase, 5 * sizeof(UC)); - val1 |= mask; - val2 |= mask; - return val1 == val2; - } - else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 2) { - mask = 0x0020002000200020; - ::memcpy(&val1, actual_mixedcase, 4 * sizeof(UC)); - ::memcpy(&val2, expected_lowercase, 4 * sizeof(UC)); - val1 |= mask; - if (val1 != val2) { - return false; - } - return (actual_mixedcase[4] | 32) == (expected_lowercase[4]); - } - else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) { - mask = 0x0000002000000020; - ::memcpy(&val1, actual_mixedcase, 2 * sizeof(UC)); - ::memcpy(&val2, expected_lowercase, 2 * sizeof(UC)); - val1 |= mask; - if (val1 != val2) { - return false; - } - ::memcpy(&val1, actual_mixedcase + 2, 2 * sizeof(UC)); - ::memcpy(&val2, expected_lowercase + 2, 2 * sizeof(UC)); - val1 |= mask; - if (val1 != val2) { - return false; - } - return (actual_mixedcase[4] | 32) == (expected_lowercase[4]); - } - else { - return false; - } - } -} - -// Compares two ASCII strings in a case insensitive manner. -template -inline SIMDJSON_FASTFLOAT_CONSTEXPR14 bool -simdjson_fastfloat_strncasecmp(UC const *actual_mixedcase, UC const *expected_lowercase, - size_t length) { - uint64_t mask{0}; - SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1) { mask = 0x2020202020202020; } - else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 2) { - mask = 0x0020002000200020; - } - else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) { - mask = 0x0000002000000020; - } - else { - return false; - } - - if (cpp20_and_in_constexpr()) { - for (size_t i = 0; i < length; i++) { - if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) { - return false; - } - } - return true; - } else { - uint64_t val1{0}, val2{0}; - size_t sz{8 / (sizeof(UC))}; - for (size_t i = 0; i < length; i += sz) { - val1 = val2 = 0; - sz = sz < (length - i) ? sz : length - i; - ::memcpy(&val1, actual_mixedcase + i, sz * sizeof(UC)); - ::memcpy(&val2, expected_lowercase + i, sz * sizeof(UC)); - val1 |= mask; - val2 |= mask; - if (val1 != val2) { - return false; - } - } - return true; - } -} - -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif - -// a pointer and a length to a contiguous block of memory -template struct span { - T const *ptr; - size_t length; - - constexpr span(T const *_ptr, size_t _length) : ptr(_ptr), length(_length) {} - - constexpr span() : ptr(nullptr), length(0) {} - - constexpr size_t len() const noexcept { return length; } - - SIMDJSON_FASTFLOAT_CONSTEXPR14 const T &operator[](size_t index) const noexcept { - SIMDJSON_FASTFLOAT_DEBUG_ASSERT(index < length); - return ptr[index]; - } -}; - -struct value128 { - uint64_t low; - uint64_t high; - - constexpr value128(uint64_t _low, uint64_t _high) : low(_low), high(_high) {} - - constexpr value128() : low(0), high(0) {} -}; - -/* Helper C++14 constexpr generic implementation of leading_zeroes */ -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 int -leading_zeroes_generic(uint64_t input_num, int last_bit = 0) { - if (input_num & uint64_t(0xffffffff00000000)) { - input_num >>= 32; - last_bit |= 32; - } - if (input_num & uint64_t(0xffff0000)) { - input_num >>= 16; - last_bit |= 16; - } - if (input_num & uint64_t(0xff00)) { - input_num >>= 8; - last_bit |= 8; - } - if (input_num & uint64_t(0xf0)) { - input_num >>= 4; - last_bit |= 4; - } - if (input_num & uint64_t(0xc)) { - input_num >>= 2; - last_bit |= 2; - } - if (input_num & uint64_t(0x2)) { /* input_num >>= 1; */ - last_bit |= 1; - } - return 63 - last_bit; -} - -/* result might be undefined when input_num is zero */ -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 int -leading_zeroes(uint64_t input_num) { - assert(input_num > 0); - if (cpp20_and_in_constexpr()) { - return leading_zeroes_generic(input_num); - } -#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO -#if defined(_M_X64) || defined(_M_ARM64) - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - _BitScanReverse64(&leading_zero, input_num); - return static_cast(63 - leading_zero); -#else - return leading_zeroes_generic(input_num); -#endif -#else - return __builtin_clzll(input_num); -#endif -} - -/* Helper C++14 constexpr generic implementation of countr_zero for 32-bit */ -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 int -countr_zero_generic_32(uint32_t input_num) { - if (input_num == 0) { - return 32; - } - int last_bit = 0; - if (!(input_num & 0x0000FFFF)) { - input_num >>= 16; - last_bit |= 16; - } - if (!(input_num & 0x00FF)) { - input_num >>= 8; - last_bit |= 8; - } - if (!(input_num & 0x0F)) { - input_num >>= 4; - last_bit |= 4; - } - if (!(input_num & 0x3)) { - input_num >>= 2; - last_bit |= 2; - } - if (!(input_num & 0x1)) { - last_bit |= 1; - } - return last_bit; -} - -/* count trailing zeroes for 32-bit integers */ -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 int -countr_zero_32(uint32_t input_num) { - if (cpp20_and_in_constexpr()) { - return countr_zero_generic_32(input_num); - } -#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO - unsigned long trailing_zero = 0; - if (_BitScanForward(&trailing_zero, input_num)) { - return static_cast(trailing_zero); - } - return 32; -#else - return input_num == 0 ? 32 : __builtin_ctz(input_num); -#endif -} - -// slow emulation routine for 32-bit -simdjson_fastfloat_really_inline constexpr uint64_t emulu(uint32_t x, uint32_t y) { - return x * static_cast(y); -} - -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 uint64_t -umul128_generic(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = - emulu(static_cast(ab >> 32), static_cast(cd)); - uint64_t bd = emulu(static_cast(ab), static_cast(cd)); - uint64_t adbc = - ad + emulu(static_cast(ab), static_cast(cd >> 32)); - uint64_t adbc_carry = static_cast(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = - emulu(static_cast(ab >> 32), static_cast(cd >> 32)) + - (adbc >> 32) + (adbc_carry << 32) + static_cast(lo < bd); - return lo; -} - -#ifdef SIMDJSON_FASTFLOAT_32BIT - -// slow emulation routine for 32-bit -#if !defined(__MINGW64__) -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 uint64_t _umul128(uint64_t ab, - uint64_t cd, - uint64_t *hi) { - return umul128_generic(ab, cd, hi); -} -#endif // !__MINGW64__ - -#endif // SIMDJSON_FASTFLOAT_32BIT - -// compute 64-bit a*b -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 value128 -full_multiplication(uint64_t a, uint64_t b) { - if (cpp20_and_in_constexpr()) { - value128 answer; - answer.low = umul128_generic(a, b, &answer.high); - return answer; - } - value128 answer; -#if defined(_M_ARM64) && !defined(__MINGW32__) - // ARM64 has native support for 64-bit multiplications, no need to emulate - // But MinGW on ARM64 doesn't have native support for 64-bit multiplications - answer.high = __umulh(a, b); - answer.low = a * b; -#elif defined(SIMDJSON_FASTFLOAT_32BIT) || (defined(_WIN64) && !defined(__clang__) && \ - !defined(_M_ARM64) && !defined(__GNUC__)) - answer.low = _umul128(a, b, &answer.high); // _umul128 not available on ARM64 -#elif defined(SIMDJSON_FASTFLOAT_64BIT) && defined(__SIZEOF_INT128__) - __uint128_t r = static_cast<__uint128_t>(a) * b; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#else - answer.low = umul128_generic(a, b, &answer.high); -#endif - return answer; -} - -struct adjusted_mantissa { - uint64_t mantissa{0}; - int32_t power2{0}; // a negative value indicates an invalid result - adjusted_mantissa() = default; - - constexpr bool operator==(adjusted_mantissa const &o) const { - return mantissa == o.mantissa && power2 == o.power2; - } - - constexpr bool operator!=(adjusted_mantissa const &o) const { - return mantissa != o.mantissa || power2 != o.power2; - } -}; - -// Bias so we can get the real exponent with an invalid adjusted_mantissa. -constexpr static int32_t invalid_am_bias = -0x8000; - -// used for binary_format_lookup_tables::max_mantissa -constexpr uint64_t constant_55555 = 5 * 5 * 5 * 5 * 5; - -template struct binary_format_lookup_tables; - -template struct binary_format : binary_format_lookup_tables { - using equiv_uint = equiv_uint_t; - - static constexpr int mantissa_explicit_bits(); - static constexpr int minimum_exponent(); - static constexpr int infinite_power(); - static constexpr int sign_index(); - static constexpr int - min_exponent_fast_path(); // used when fegetround() == FE_TONEAREST - static constexpr int max_exponent_fast_path(); - static constexpr int max_exponent_round_to_even(); - static constexpr int min_exponent_round_to_even(); - static constexpr uint64_t max_mantissa_fast_path(int64_t power); - static constexpr uint64_t - max_mantissa_fast_path(); // used when fegetround() == FE_TONEAREST - static constexpr int largest_power_of_ten(); - static constexpr int smallest_power_of_ten(); - static constexpr T exact_power_of_ten(int64_t power); - static constexpr size_t max_digits(); - static constexpr equiv_uint exponent_mask(); - static constexpr equiv_uint mantissa_mask(); - static constexpr equiv_uint hidden_bit_mask(); -}; - -template struct binary_format_lookup_tables { - static constexpr double powers_of_ten[] = { - 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, - 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22}; - - // Largest integer value v so that (5**index * v) <= 1<<53. - // 0x20000000000000 == 1 << 53 - static constexpr uint64_t max_mantissa[] = { - 0x20000000000000, - 0x20000000000000 / 5, - 0x20000000000000 / (5 * 5), - 0x20000000000000 / (5 * 5 * 5), - 0x20000000000000 / (5 * 5 * 5 * 5), - 0x20000000000000 / (constant_55555), - 0x20000000000000 / (constant_55555 * 5), - 0x20000000000000 / (constant_55555 * 5 * 5), - 0x20000000000000 / (constant_55555 * 5 * 5 * 5), - 0x20000000000000 / (constant_55555 * 5 * 5 * 5 * 5), - 0x20000000000000 / (constant_55555 * constant_55555), - 0x20000000000000 / (constant_55555 * constant_55555 * 5), - 0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5), - 0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5 * 5), - 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555), - 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * 5), - 0x20000000000000 / - (constant_55555 * constant_55555 * constant_55555 * 5 * 5), - 0x20000000000000 / - (constant_55555 * constant_55555 * constant_55555 * 5 * 5 * 5), - 0x20000000000000 / - (constant_55555 * constant_55555 * constant_55555 * 5 * 5 * 5 * 5), - 0x20000000000000 / - (constant_55555 * constant_55555 * constant_55555 * constant_55555), - 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * - constant_55555 * 5), - 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * - constant_55555 * 5 * 5), - 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * - constant_55555 * 5 * 5 * 5), - 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * - constant_55555 * 5 * 5 * 5 * 5)}; -}; - -#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE - -template -constexpr double binary_format_lookup_tables::powers_of_ten[]; - -template -constexpr uint64_t binary_format_lookup_tables::max_mantissa[]; - -#endif - -template struct binary_format_lookup_tables { - static constexpr float powers_of_ten[] = {1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - - // Largest integer value v so that (5**index * v) <= 1<<24. - // 0x1000000 == 1<<24 - static constexpr uint64_t max_mantissa[] = { - 0x1000000, - 0x1000000 / 5, - 0x1000000 / (5 * 5), - 0x1000000 / (5 * 5 * 5), - 0x1000000 / (5 * 5 * 5 * 5), - 0x1000000 / (constant_55555), - 0x1000000 / (constant_55555 * 5), - 0x1000000 / (constant_55555 * 5 * 5), - 0x1000000 / (constant_55555 * 5 * 5 * 5), - 0x1000000 / (constant_55555 * 5 * 5 * 5 * 5), - 0x1000000 / (constant_55555 * constant_55555), - 0x1000000 / (constant_55555 * constant_55555 * 5)}; -}; - -#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE - -template -constexpr float binary_format_lookup_tables::powers_of_ten[]; - -template -constexpr uint64_t binary_format_lookup_tables::max_mantissa[]; - -#endif - -template <> -inline constexpr int binary_format::min_exponent_fast_path() { -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - return 0; -#else - return -22; -#endif -} - -template <> -inline constexpr int binary_format::min_exponent_fast_path() { -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - return 0; -#else - return -10; -#endif -} - -template <> -inline constexpr int binary_format::mantissa_explicit_bits() { - return 52; -} - -template <> -inline constexpr int binary_format::mantissa_explicit_bits() { - return 23; -} - -template <> -inline constexpr int binary_format::max_exponent_round_to_even() { - return 23; -} - -template <> -inline constexpr int binary_format::max_exponent_round_to_even() { - return 10; -} - -template <> -inline constexpr int binary_format::min_exponent_round_to_even() { - return -4; -} - -template <> -inline constexpr int binary_format::min_exponent_round_to_even() { - return -17; -} - -template <> inline constexpr int binary_format::minimum_exponent() { - return -1023; -} - -template <> inline constexpr int binary_format::minimum_exponent() { - return -127; -} - -template <> inline constexpr int binary_format::infinite_power() { - return 0x7FF; -} - -template <> inline constexpr int binary_format::infinite_power() { - return 0xFF; -} - -template <> inline constexpr int binary_format::sign_index() { - return 63; -} - -template <> inline constexpr int binary_format::sign_index() { - return 31; -} - -template <> -inline constexpr int binary_format::max_exponent_fast_path() { - return 22; -} - -template <> -inline constexpr int binary_format::max_exponent_fast_path() { - return 10; -} - -template <> -inline constexpr uint64_t binary_format::max_mantissa_fast_path() { - return uint64_t(2) << mantissa_explicit_bits(); -} - -template <> -inline constexpr uint64_t binary_format::max_mantissa_fast_path() { - return uint64_t(2) << mantissa_explicit_bits(); -} - -// credit: Jakub Jelinek -#ifdef __STDCPP_FLOAT16_T__ -template struct binary_format_lookup_tables { - static constexpr std::float16_t powers_of_ten[] = {1e0f16, 1e1f16, 1e2f16, - 1e3f16, 1e4f16}; - - // Largest integer value v so that (5**index * v) <= 1<<11. - // 0x800 == 1<<11 - static constexpr uint64_t max_mantissa[] = {0x800, - 0x800 / 5, - 0x800 / (5 * 5), - 0x800 / (5 * 5 * 5), - 0x800 / (5 * 5 * 5 * 5), - 0x800 / (constant_55555)}; -}; - -#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE - -template -constexpr std::float16_t - binary_format_lookup_tables::powers_of_ten[]; - -template -constexpr uint64_t - binary_format_lookup_tables::max_mantissa[]; - -#endif - -template <> -inline constexpr std::float16_t -binary_format::exact_power_of_ten(int64_t power) { - // Work around clang bug https://godbolt.org/z/zedh7rrhc - return static_cast(powers_of_ten[0]), powers_of_ten[power]; -} - -template <> -inline constexpr binary_format::equiv_uint -binary_format::exponent_mask() { - return 0x7C00; -} - -template <> -inline constexpr binary_format::equiv_uint -binary_format::mantissa_mask() { - return 0x03FF; -} - -template <> -inline constexpr binary_format::equiv_uint -binary_format::hidden_bit_mask() { - return 0x0400; -} - -template <> -inline constexpr int binary_format::max_exponent_fast_path() { - return 4; -} - -template <> -inline constexpr int binary_format::mantissa_explicit_bits() { - return 10; -} - -template <> -inline constexpr uint64_t -binary_format::max_mantissa_fast_path() { - return uint64_t(2) << mantissa_explicit_bits(); -} - -template <> -inline constexpr uint64_t -binary_format::max_mantissa_fast_path(int64_t power) { - // caller is responsible to ensure that - // power >= 0 && power <= 4 - // - // Work around clang bug https://godbolt.org/z/zedh7rrhc - return static_cast(max_mantissa[0]), max_mantissa[power]; -} - -template <> -inline constexpr int binary_format::min_exponent_fast_path() { - return 0; -} - -template <> -inline constexpr int -binary_format::max_exponent_round_to_even() { - return 5; -} - -template <> -inline constexpr int -binary_format::min_exponent_round_to_even() { - return -22; -} - -template <> -inline constexpr int binary_format::minimum_exponent() { - return -15; -} - -template <> -inline constexpr int binary_format::infinite_power() { - return 0x1F; -} - -template <> inline constexpr int binary_format::sign_index() { - return 15; -} - -template <> -inline constexpr int binary_format::largest_power_of_ten() { - return 4; -} - -template <> -inline constexpr int binary_format::smallest_power_of_ten() { - return -27; -} - -template <> -inline constexpr size_t binary_format::max_digits() { - return 22; -} -#endif // __STDCPP_FLOAT16_T__ - -// credit: Jakub Jelinek -#ifdef __STDCPP_BFLOAT16_T__ -template struct binary_format_lookup_tables { - static constexpr std::bfloat16_t powers_of_ten[] = {1e0bf16, 1e1bf16, 1e2bf16, - 1e3bf16}; - - // Largest integer value v so that (5**index * v) <= 1<<8. - // 0x100 == 1<<8 - static constexpr uint64_t max_mantissa[] = {0x100, 0x100 / 5, 0x100 / (5 * 5), - 0x100 / (5 * 5 * 5), - 0x100 / (5 * 5 * 5 * 5)}; -}; - -#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE - -template -constexpr std::bfloat16_t - binary_format_lookup_tables::powers_of_ten[]; - -template -constexpr uint64_t - binary_format_lookup_tables::max_mantissa[]; - -#endif - -template <> -inline constexpr std::bfloat16_t -binary_format::exact_power_of_ten(int64_t power) { - // Work around clang bug https://godbolt.org/z/zedh7rrhc - return static_cast(powers_of_ten[0]), powers_of_ten[power]; -} - -template <> -inline constexpr int binary_format::max_exponent_fast_path() { - return 3; -} - -template <> -inline constexpr binary_format::equiv_uint -binary_format::exponent_mask() { - return 0x7F80; -} - -template <> -inline constexpr binary_format::equiv_uint -binary_format::mantissa_mask() { - return 0x007F; -} - -template <> -inline constexpr binary_format::equiv_uint -binary_format::hidden_bit_mask() { - return 0x0080; -} - -template <> -inline constexpr int binary_format::mantissa_explicit_bits() { - return 7; -} - -template <> -inline constexpr uint64_t -binary_format::max_mantissa_fast_path() { - return uint64_t(2) << mantissa_explicit_bits(); -} - -template <> -inline constexpr uint64_t -binary_format::max_mantissa_fast_path(int64_t power) { - // caller is responsible to ensure that - // power >= 0 && power <= 3 - // - // Work around clang bug https://godbolt.org/z/zedh7rrhc - return static_cast(max_mantissa[0]), max_mantissa[power]; -} - -template <> -inline constexpr int binary_format::min_exponent_fast_path() { - return 0; -} - -template <> -inline constexpr int -binary_format::max_exponent_round_to_even() { - return 3; -} - -template <> -inline constexpr int -binary_format::min_exponent_round_to_even() { - return -24; -} - -template <> -inline constexpr int binary_format::minimum_exponent() { - return -127; -} - -template <> -inline constexpr int binary_format::infinite_power() { - return 0xFF; -} - -template <> inline constexpr int binary_format::sign_index() { - return 15; -} - -template <> -inline constexpr int binary_format::largest_power_of_ten() { - return 38; -} - -template <> -inline constexpr int binary_format::smallest_power_of_ten() { - return -60; -} - -template <> -inline constexpr size_t binary_format::max_digits() { - return 98; -} -#endif // __STDCPP_BFLOAT16_T__ - -template <> -inline constexpr uint64_t -binary_format::max_mantissa_fast_path(int64_t power) { - // caller is responsible to ensure that - // power >= 0 && power <= 22 - // - // Work around clang bug https://godbolt.org/z/zedh7rrhc - return static_cast(max_mantissa[0]), max_mantissa[power]; -} - -template <> -inline constexpr uint64_t -binary_format::max_mantissa_fast_path(int64_t power) { - // caller is responsible to ensure that - // power >= 0 && power <= 10 - // - // Work around clang bug https://godbolt.org/z/zedh7rrhc - return static_cast(max_mantissa[0]), max_mantissa[power]; -} - -template <> -inline constexpr double -binary_format::exact_power_of_ten(int64_t power) { - // Work around clang bug https://godbolt.org/z/zedh7rrhc - return static_cast(powers_of_ten[0]), powers_of_ten[power]; -} - -template <> -inline constexpr float binary_format::exact_power_of_ten(int64_t power) { - // Work around clang bug https://godbolt.org/z/zedh7rrhc - return static_cast(powers_of_ten[0]), powers_of_ten[power]; -} - -template <> inline constexpr int binary_format::largest_power_of_ten() { - return 308; -} - -template <> inline constexpr int binary_format::largest_power_of_ten() { - return 38; -} - -template <> -inline constexpr int binary_format::smallest_power_of_ten() { - return -342; -} - -template <> inline constexpr int binary_format::smallest_power_of_ten() { - return -64; -} - -template <> inline constexpr size_t binary_format::max_digits() { - return 769; -} - -template <> inline constexpr size_t binary_format::max_digits() { - return 114; -} - -template <> -inline constexpr binary_format::equiv_uint -binary_format::exponent_mask() { - return 0x7F800000; -} - -template <> -inline constexpr binary_format::equiv_uint -binary_format::exponent_mask() { - return 0x7FF0000000000000; -} - -template <> -inline constexpr binary_format::equiv_uint -binary_format::mantissa_mask() { - return 0x007FFFFF; -} - -template <> -inline constexpr binary_format::equiv_uint -binary_format::mantissa_mask() { - return 0x000FFFFFFFFFFFFF; -} - -template <> -inline constexpr binary_format::equiv_uint -binary_format::hidden_bit_mask() { - return 0x00800000; -} - -template <> -inline constexpr binary_format::equiv_uint -binary_format::hidden_bit_mask() { - return 0x0010000000000000; -} - -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void -to_float(bool negative, adjusted_mantissa am, T &value) { - using equiv_uint = equiv_uint_t; - equiv_uint word = equiv_uint(am.mantissa); - word = equiv_uint(word | equiv_uint(am.power2) - << binary_format::mantissa_explicit_bits()); - word = - equiv_uint(word | equiv_uint(negative) << binary_format::sign_index()); -#if SIMDJSON_FASTFLOAT_HAS_BIT_CAST - value = std::bit_cast(word); -#else - ::memcpy(&value, &word, sizeof(T)); -#endif -} - -template struct space_lut { - static constexpr bool value[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; -}; - -#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE - -template constexpr bool space_lut::value[]; - -#endif - -template constexpr bool is_space(UC c) { - // wchar_t and char can be signed, so a negative code unit slips past a plain - // `c < 256` and then indexes the table by its truncated low byte. Compare as - // unsigned, matching the care taken in ch_to_digit. - using UnsignedUC = typename std::make_unsigned::type; - return static_cast(c) < 256 && space_lut<>::value[uint8_t(c)]; -} - -template static constexpr uint64_t int_cmp_zeros() { - static_assert((sizeof(UC) == 1) || (sizeof(UC) == 2) || (sizeof(UC) == 4), - "Unsupported character size"); - return (sizeof(UC) == 1) ? 0x3030303030303030 - : (sizeof(UC) == 2) - ? (uint64_t(UC('0')) << 48 | uint64_t(UC('0')) << 32 | - uint64_t(UC('0')) << 16 | UC('0')) - : (uint64_t(UC('0')) << 32 | UC('0')); -} - -template static constexpr int int_cmp_len() { - return sizeof(uint64_t) / sizeof(UC); -} - -template constexpr UC const *str_const_nan(); - -template <> constexpr char const *str_const_nan() { return "nan"; } - -template <> constexpr wchar_t const *str_const_nan() { return L"nan"; } - -template <> constexpr char16_t const *str_const_nan() { - return u"nan"; -} - -template <> constexpr char32_t const *str_const_nan() { - return U"nan"; -} - -#ifdef __cpp_char8_t -template <> constexpr char8_t const *str_const_nan() { - return u8"nan"; -} -#endif - -template constexpr UC const *str_const_inf(); - -template <> constexpr char const *str_const_inf() { return "infinity"; } - -template <> constexpr wchar_t const *str_const_inf() { - return L"infinity"; -} - -template <> constexpr char16_t const *str_const_inf() { - return u"infinity"; -} - -template <> constexpr char32_t const *str_const_inf() { - return U"infinity"; -} - -#ifdef __cpp_char8_t -template <> constexpr char8_t const *str_const_inf() { - return u8"infinity"; -} -#endif - -template struct int_luts { - static constexpr uint8_t chdigit[] = { - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, - 255, 255, 255, 255, 255, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, - 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, - 35, 255, 255, 255, 255, 255, 255, 10, 11, 12, 13, 14, 15, 16, 17, - 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, - 33, 34, 35, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255}; - - static constexpr size_t maxdigits_u64[] = { - 64, 41, 32, 28, 25, 23, 22, 21, 20, 19, 18, 18, 17, 17, 16, 16, 16, 16, - 15, 15, 15, 15, 14, 14, 14, 14, 14, 14, 14, 13, 13, 13, 13, 13, 13}; - - static constexpr uint64_t min_safe_u64[] = { - 9223372036854775808ull, 12157665459056928801ull, 4611686018427387904, - 7450580596923828125, 4738381338321616896, 3909821048582988049, - 9223372036854775808ull, 12157665459056928801ull, 10000000000000000000ull, - 5559917313492231481, 2218611106740436992, 8650415919381337933, - 2177953337809371136, 6568408355712890625, 1152921504606846976, - 2862423051509815793, 6746640616477458432, 15181127029874798299ull, - 1638400000000000000, 3243919932521508681, 6221821273427820544, - 11592836324538749809ull, 876488338465357824, 1490116119384765625, - 2481152873203736576, 4052555153018976267, 6502111422497947648, - 10260628712958602189ull, 15943230000000000000ull, 787662783788549761, - 1152921504606846976, 1667889514952984961, 2386420683693101056, - 3379220508056640625, 4738381338321616896}; -}; - -#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE - -template constexpr uint8_t int_luts::chdigit[]; - -template constexpr size_t int_luts::maxdigits_u64[]; - -template constexpr uint64_t int_luts::min_safe_u64[]; - -#endif - -template -simdjson_fastfloat_really_inline constexpr uint8_t ch_to_digit(UC c) { - // wchar_t and char can be signed, so we need to be careful. - using UnsignedUC = typename std::make_unsigned::type; - return int_luts<>::chdigit[static_cast( - static_cast(c) & - static_cast( - -((static_cast(c) & ~0xFFull) == 0)))]; -} - -simdjson_fastfloat_really_inline constexpr size_t max_digits_u64(int base) { - return int_luts<>::maxdigits_u64[base - 2]; -} - -// If a u64 is exactly max_digits_u64() in length, this is -// the value below which it has definitely overflowed. -simdjson_fastfloat_really_inline constexpr uint64_t min_safe_u64(int base) { - return int_luts<>::min_safe_u64[base - 2]; -} - -static_assert(std::is_same, uint64_t>::value, - "equiv_uint should be uint64_t for double"); -static_assert(std::numeric_limits::is_iec559, - "double must fulfill the requirements of IEC 559 (IEEE 754)"); - -static_assert(std::is_same, uint32_t>::value, - "equiv_uint should be uint32_t for float"); -static_assert(std::numeric_limits::is_iec559, - "float must fulfill the requirements of IEC 559 (IEEE 754)"); - -#ifdef __STDCPP_FLOAT64_T__ -static_assert(std::is_same, uint64_t>::value, - "equiv_uint should be uint64_t for std::float64_t"); -static_assert( - std::numeric_limits::is_iec559, - "std::float64_t must fulfill the requirements of IEC 559 (IEEE 754)"); - -template <> -struct binary_format : public binary_format {}; -#endif // __STDCPP_FLOAT64_T__ - -#ifdef __STDCPP_FLOAT32_T__ -static_assert(std::is_same, uint32_t>::value, - "equiv_uint should be uint32_t for std::float32_t"); -static_assert( - std::numeric_limits::is_iec559, - "std::float32_t must fulfill the requirements of IEC 559 (IEEE 754)"); - -template <> -struct binary_format : public binary_format {}; -#endif // __STDCPP_FLOAT32_T__ - -#ifdef __STDCPP_FLOAT16_T__ -static_assert( - std::is_same::equiv_uint, uint16_t>::value, - "equiv_uint should be uint16_t for std::float16_t"); -static_assert( - std::numeric_limits::is_iec559, - "std::float16_t must fulfill the requirements of IEC 559 (IEEE 754)"); -#endif // __STDCPP_FLOAT16_T__ - -#ifdef __STDCPP_BFLOAT16_T__ -static_assert( - std::is_same::equiv_uint, uint16_t>::value, - "equiv_uint should be uint16_t for std::bfloat16_t"); -static_assert( - std::numeric_limits::is_iec559, - "std::bfloat16_t must fulfill the requirements of IEC 559 (IEEE 754)"); -#endif // __STDCPP_BFLOAT16_T__ - -constexpr chars_format operator~(chars_format rhs) noexcept { - using int_type = std::underlying_type::type; - return static_cast(~static_cast(rhs)); -} - -constexpr chars_format operator&(chars_format lhs, chars_format rhs) noexcept { - using int_type = std::underlying_type::type; - return static_cast(static_cast(lhs) & - static_cast(rhs)); -} - -constexpr chars_format operator|(chars_format lhs, chars_format rhs) noexcept { - using int_type = std::underlying_type::type; - return static_cast(static_cast(lhs) | - static_cast(rhs)); -} - -constexpr chars_format operator^(chars_format lhs, chars_format rhs) noexcept { - using int_type = std::underlying_type::type; - return static_cast(static_cast(lhs) ^ - static_cast(rhs)); -} - -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 chars_format & -operator&=(chars_format &lhs, chars_format rhs) noexcept { - return lhs = (lhs & rhs); -} - -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 chars_format & -operator|=(chars_format &lhs, chars_format rhs) noexcept { - return lhs = (lhs | rhs); -} - -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 chars_format & -operator^=(chars_format &lhs, chars_format rhs) noexcept { - return lhs = (lhs ^ rhs); -} - -namespace detail { -// adjust for deprecated feature macros -constexpr chars_format adjust_for_feature_macros(chars_format fmt) { - return fmt -#ifdef SIMDJSON_FASTFLOAT_ALLOWS_LEADING_PLUS - | chars_format::allow_leading_plus -#endif -#ifdef SIMDJSON_FASTFLOAT_SKIP_WHITE_SPACE - | chars_format::skip_white_space -#endif - ; -} -} // namespace detail -} // namespace simdjson_fast_float - -#endif - - -#ifndef SIMDJSON_FASTFLOAT_FAST_FLOAT_H -#define SIMDJSON_FASTFLOAT_FAST_FLOAT_H - - -namespace simdjson_fast_float { -/** - * This function parses the character sequence [first,last) for a number. It - * parses floating-point numbers expecting a locale-independent format - * equivalent to what is used by std::strtod in the default ("C") locale. The - * resulting floating-point value is the closest floating-point values (using - * either float or double), using the "round to even" convention for values that - * would otherwise fall right in-between two values. That is, we provide exact - * parsing according to the IEEE standard. - * - * Given a successful parse, the pointer (`ptr`) in the returned value is set to - * point right after the parsed number, and the `value` referenced is set to the - * parsed value. In case of error, the returned `ec` contains a representative - * error, otherwise the default (`std::errc()`) value is stored. - * - * The implementation does not throw and does not allocate memory (e.g., with - * `new` or `malloc`). - * - * Like the C++17 standard, the `simdjson_fast_float::from_chars` functions take an - * optional last argument of the type `simdjson_fast_float::chars_format`. It is a bitset - * value: we check whether `fmt & simdjson_fast_float::chars_format::fixed` and `fmt & - * simdjson_fast_float::chars_format::scientific` are set to determine whether we allow - * the fixed point and scientific notation respectively. The default is - * `simdjson_fast_float::chars_format::general` which allows both `fixed` and - * `scientific`. - */ -template ::value)> -SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars(UC const *first, UC const *last, T &value, - chars_format fmt = chars_format::general) noexcept; - -/** - * Like from_chars, but accepts an `options` argument to govern number parsing. - * Both for floating-point types and integer types. - */ -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars_advanced(UC const *first, UC const *last, T &value, - parse_options_t options) noexcept; - -/** - * This function multiplies an integer number by a power of 10 and returns - * the result as a double precision floating-point value that is correctly - * rounded. The resulting floating-point value is the closest floating-point - * value, using the "round to nearest, tie to even" convention for values that - * would otherwise fall right in-between two values. That is, we provide exact - * conversion according to the IEEE standard. - * - * On overflow infinity is returned, on underflow 0 is returned. - * - * The implementation does not throw and does not allocate memory (e.g., with - * `new` or `malloc`). - */ -SIMDJSON_FASTFLOAT_CONSTEXPR20 inline double -integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept; -SIMDJSON_FASTFLOAT_CONSTEXPR20 inline double -integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept; - -/** - * This function is a template overload of `integer_times_pow10()` - * that returns a floating-point value of type `T` that is one of - * supported floating-point types (e.g. `double`, `float`). - */ -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 - typename std::enable_if::value, T>::type - integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept; -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 - typename std::enable_if::value, T>::type - integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept; - -/** - * from_chars for integer types. - */ -template ::value)> -SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars(UC const *first, UC const *last, T &value, int base = 10) noexcept; - -} // namespace simdjson_fast_float - -#endif // SIMDJSON_FASTFLOAT_FAST_FLOAT_H - -#ifndef SIMDJSON_FASTFLOAT_ASCII_NUMBER_H -#define SIMDJSON_FASTFLOAT_ASCII_NUMBER_H - -#include -#include -#include -#include -#include -#include - - -#ifdef SIMDJSON_FASTFLOAT_SSE2 -#include -#endif - -#ifdef SIMDJSON_FASTFLOAT_NEON -#include -#endif - -namespace simdjson_fast_float { - -template simdjson_fastfloat_really_inline constexpr bool has_simd_opt() { -#ifdef SIMDJSON_FASTFLOAT_HAS_SIMD - return std::is_same::value; -#else - return false; -#endif -} - -// Next function can be micro-optimized, but compilers are entirely -// able to optimize it well. -template -simdjson_fastfloat_really_inline constexpr bool is_integer(UC c) noexcept { - return static_cast(c - UC('0')) <= 9u; -} - -simdjson_fastfloat_really_inline constexpr uint64_t byteswap(uint64_t val) { - return (val & 0xFF00000000000000) >> 56 | (val & 0x00FF000000000000) >> 40 | - (val & 0x0000FF0000000000) >> 24 | (val & 0x000000FF00000000) >> 8 | - (val & 0x00000000FF000000) << 8 | (val & 0x0000000000FF0000) << 24 | - (val & 0x000000000000FF00) << 40 | (val & 0x00000000000000FF) << 56; -} - -simdjson_fastfloat_really_inline constexpr uint32_t byteswap_32(uint32_t val) { - return (val >> 24) | ((val >> 8) & 0x0000FF00u) | ((val << 8) & 0x00FF0000u) | - (val << 24); -} - -// Read 8 UC into a u64. Truncates UC if not char. -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t -read8_to_u64(UC const *chars) { - if (cpp20_and_in_constexpr() || !std::is_same::value) { - uint64_t val = 0; - for (int i = 0; i < 8; ++i) { - val |= uint64_t(uint8_t(*chars)) << (i * 8); - ++chars; - } - return val; - } - uint64_t val; - ::memcpy(&val, chars, sizeof(uint64_t)); -#if SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN == 1 - // Need to read as-if the number was in little-endian order. - val = byteswap(val); -#endif - return val; -} - -// Read 4 UC into a u32. Truncates UC if not char. -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint32_t -read4_to_u32(UC const *chars) { - if (cpp20_and_in_constexpr() || !std::is_same::value) { - uint32_t val = 0; - for (int i = 0; i < 4; ++i) { - val |= uint32_t(uint8_t(*chars)) << (i * 8); - ++chars; - } - return val; - } - uint32_t val; - ::memcpy(&val, chars, sizeof(uint32_t)); -#if SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN == 1 - val = byteswap_32(val); -#endif - return val; -} -#ifdef SIMDJSON_FASTFLOAT_SSE2 - -simdjson_fastfloat_really_inline uint64_t simd_read8_to_u64(__m128i const data) { - SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS - __m128i const packed = _mm_packus_epi16(data, data); -#ifdef SIMDJSON_FASTFLOAT_64BIT - return uint64_t(_mm_cvtsi128_si64(packed)); -#else - uint64_t value; - // Visual Studio + older versions of GCC don't support _mm_storeu_si64 - _mm_storel_epi64(reinterpret_cast<__m128i *>(&value), packed); - return value; -#endif - SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS -} - -simdjson_fastfloat_really_inline uint64_t simd_read8_to_u64(char16_t const *chars) { - SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS - return simd_read8_to_u64( - _mm_loadu_si128(reinterpret_cast<__m128i const *>(chars))); - SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS -} - -#elif defined(SIMDJSON_FASTFLOAT_NEON) - -simdjson_fastfloat_really_inline uint64_t simd_read8_to_u64(uint16x8_t const data) { - SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS - uint8x8_t utf8_packed = vmovn_u16(data); - return vget_lane_u64(vreinterpret_u64_u8(utf8_packed), 0); - SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS -} - -simdjson_fastfloat_really_inline uint64_t simd_read8_to_u64(char16_t const *chars) { - SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS - return simd_read8_to_u64( - vld1q_u16(reinterpret_cast(chars))); - SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS -} - -#endif // SIMDJSON_FASTFLOAT_SSE2 - -// MSVC SFINAE is broken pre-VS2017 -#if defined(_MSC_VER) && _MSC_VER <= 1900 -template -#else -template ()) = 0> -#endif -// dummy for compile -uint64_t simd_read8_to_u64(UC const *) { - return 0; -} - -// credit @aqrit -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 uint32_t -parse_eight_digits_unrolled(uint64_t val) { - uint64_t const mask = 0x000000FF000000FF; - uint64_t const mul1 = 0x000F424000000064; // 100 + (1000000ULL << 32) - uint64_t const mul2 = 0x0000271000000001; // 1 + (10000ULL << 32) - val -= 0x3030303030303030; - val = (val * 10) + (val >> 8); // val = (val * 2561) >> 8; - val = (((val & mask) * mul1) + (((val >> 16) & mask) * mul2)) >> 32; - return uint32_t(val); -} - -// Call this if chars are definitely 8 digits. -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint32_t -parse_eight_digits_unrolled(UC const *chars) noexcept { - if (cpp20_and_in_constexpr() || !has_simd_opt()) { - return parse_eight_digits_unrolled(read8_to_u64(chars)); // truncation okay - } - return parse_eight_digits_unrolled(simd_read8_to_u64(chars)); -} - -// credit @aqrit -simdjson_fastfloat_really_inline constexpr bool -is_made_of_eight_digits_fast(uint64_t val) noexcept { - return !((((val + 0x4646464646464646) | (val - 0x3030303030303030)) & - 0x8080808080808080)); -} - -simdjson_fastfloat_really_inline constexpr bool -is_made_of_four_digits_fast(uint32_t val) noexcept { - return !((((val + 0x46464646) | (val - 0x30303030)) & 0x80808080)); -} - -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 uint32_t -parse_four_digits_unrolled(uint32_t val) noexcept { - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -#ifdef SIMDJSON_FASTFLOAT_HAS_SIMD - -// Call this if chars might not be 8 digits. -// Using this style (instead of is_made_of_eight_digits_fast() then -// parse_eight_digits_unrolled()) ensures we don't load SIMD registers twice. -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool -simd_parse_if_eight_digits_unrolled(char16_t const *chars, - uint64_t &i) noexcept { - if (cpp20_and_in_constexpr()) { - return false; - } -#ifdef SIMDJSON_FASTFLOAT_SSE2 - SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS - __m128i const data = - _mm_loadu_si128(reinterpret_cast<__m128i const *>(chars)); - - // (x - '0') <= 9 - // http://0x80.pl/articles/simd-parsing-int-sequences.html - __m128i const t0 = _mm_add_epi16(data, _mm_set1_epi16(32720)); - __m128i const t1 = _mm_cmpgt_epi16(t0, _mm_set1_epi16(-32759)); - - if (_mm_movemask_epi8(t1) == 0) { - i = i * 100000000 + parse_eight_digits_unrolled(simd_read8_to_u64(data)); - return true; - } else - return false; - SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS -#elif defined(SIMDJSON_FASTFLOAT_NEON) - SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS - uint16x8_t const data = vld1q_u16(reinterpret_cast(chars)); - - // (x - '0') <= 9 - // http://0x80.pl/articles/simd-parsing-int-sequences.html - uint16x8_t const t0 = vsubq_u16(data, vmovq_n_u16('0')); - uint16x8_t const mask = vcltq_u16(t0, vmovq_n_u16('9' - '0' + 1)); - - if (vminvq_u16(mask) == 0xFFFF) { - i = i * 100000000 + parse_eight_digits_unrolled(simd_read8_to_u64(data)); - return true; - } else - return false; - SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS -#else - static_cast(chars); - static_cast(i); - return false; -#endif // SIMDJSON_FASTFLOAT_SSE2 -} - -#endif // SIMDJSON_FASTFLOAT_HAS_SIMD - -// MSVC SFINAE is broken pre-VS2017 -#if defined(_MSC_VER) && _MSC_VER <= 1900 -template -#else -template ()) = 0> -#endif -// dummy for compile -bool simd_parse_if_eight_digits_unrolled(UC const *, uint64_t &) { - return 0; -} - -template ::value) = 0> -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void -loop_parse_if_eight_digits(UC const *&p, UC const *const pend, uint64_t &i) { - if (!has_simd_opt()) { - return; - } - while ((std::distance(p, pend) >= 8) && - simd_parse_if_eight_digits_unrolled( - p, i)) { // in rare cases, this will overflow, but that's ok - p += 8; - } -} - -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void -loop_parse_if_eight_digits(char const *&p, char const *const pend, - uint64_t &i) { - // optimizes better than parse_if_eight_digits_unrolled() for UC = char. - while ((std::distance(p, pend) >= 8) && - is_made_of_eight_digits_fast(read8_to_u64(p))) { - i = i * 100000000 + - parse_eight_digits_unrolled(read8_to_u64( - p)); // in rare cases, this will overflow, but that's ok - p += 8; - } - // Consume a remaining 4-7 digit run in a single SWAR step instead of - // byte-by-byte (reuses the existing 4-digit helpers). The parsed result is - // identical either way. Historically gated to clang because gcc regressed on - // short remainders, but that verdict predates the span-elision restructure; - // with the leaner hot path the 4-digit step now wins on gcc as well. - if ((pend - p) >= 4) { - uint32_t const val4 = read4_to_u32(p); - if (is_made_of_four_digits_fast(val4)) { - i = i * 10000 + - parse_four_digits_unrolled(val4); // may overflow, that's ok - p += 4; - } - } -} - -enum class parse_error { - no_error, - // [JSON-only] The minus sign must be followed by an integer. - missing_integer_after_sign, - // A sign must be followed by an integer or dot. - missing_integer_or_dot_after_sign, - // [JSON-only] The integer part must not have leading zeros. - leading_zeros_in_integer_part, - // [JSON-only] The integer part must have at least one digit. - no_digits_in_integer_part, - // [JSON-only] If there is a decimal point, there must be digits in the - // fractional part. - no_digits_in_fractional_part, - // The mantissa must have at least one digit. - no_digits_in_mantissa, - // Scientific notation requires an exponential part. - missing_exponential_part, -}; - -template struct parsed_number_string_t { - int64_t exponent{0}; - uint64_t mantissa{0}; - UC const *lastmatch{nullptr}; - bool negative{false}; - bool valid{false}; - bool too_many_digits{false}; - // contains the range of the significant digits - span integer{}; // non-nullable - span fraction{}; // nullable - parse_error error{parse_error::no_error}; -}; - -using byte_span = span; -using parsed_number_string = parsed_number_string_t; - -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 parsed_number_string_t -report_parse_error(UC const *p, parse_error error) { - parsed_number_string_t answer; - answer.valid = false; - answer.lastmatch = p; - answer.error = error; - return answer; -} - -// Assuming that you use no more than 19 digits, this will -// parse an ASCII string. -// -// store_spans is a *runtime* flag (not a template parameter, deliberately: a -// template would create a second instantiation of this whole function and the -// extra icache pressure wipes out the gain). When false, the integer/fraction -// spans (read only by the rare digit_comp slow path) are not materialized, -// which keeps the fat parsed_number_string_t off the hot path. The caller -// re-parses with store_spans=true if the slow path is actually reached. -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 parsed_number_string_t -parse_number_string(UC const *p, UC const *pend, parse_options_t options, - bool store_spans = true) noexcept { - chars_format const fmt = detail::adjust_for_feature_macros(options.format); - UC const decimal_point = options.decimal_point; - - parsed_number_string_t answer; - answer.valid = false; - answer.too_many_digits = false; - // assume p < pend, so dereference without checks; - answer.negative = (*p == UC('-')); - // C++17 20.19.3.(7.1) explicitly forbids '+' sign here - if ((*p == UC('-')) || (uint64_t(fmt & chars_format::allow_leading_plus) && - !basic_json_fmt && *p == UC('+'))) { - ++p; - if (p == pend) { - return report_parse_error( - p, parse_error::missing_integer_or_dot_after_sign); - } - SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(basic_json_fmt) { - if (!is_integer(*p)) { // a sign must be followed by an integer - return report_parse_error(p, - parse_error::missing_integer_after_sign); - } - } - else { - if (!is_integer(*p) && - (*p != - decimal_point)) { // a sign must be followed by an integer or the dot - return report_parse_error( - p, parse_error::missing_integer_or_dot_after_sign); - } - } - } - UC const *const start_digits = p; - - uint64_t i = 0; // an unsigned int avoids signed overflows (which are bad) - - // Straight-line unroll of the integer-part scan: most integer parts are - // 1-5 digits, so peeling the first iterations eliminates the loop back-edge - // for the common case. Semantics are identical to the original `while` loop: - // i = 10*i + digit, advancing p. - if ((p != pend) && is_integer(*p)) { - i = uint64_t(*p - UC('0')); - ++p; - if ((p != pend) && is_integer(*p)) { - i = 10 * i + uint64_t(*p - UC('0')); - ++p; - if ((p != pend) && is_integer(*p)) { - i = 10 * i + uint64_t(*p - UC('0')); - ++p; - if ((p != pend) && is_integer(*p)) { - i = 10 * i + uint64_t(*p - UC('0')); - ++p; - if ((p != pend) && is_integer(*p)) { - i = 10 * i + uint64_t(*p - UC('0')); - ++p; - while ((p != pend) && is_integer(*p)) { - // a multiplication by 10 is cheaper than an arbitrary integer - // multiplication - i = 10 * i + - uint64_t(*p - UC('0')); // might overflow, handled later - ++p; - } - } - } - } - } - } - UC const *const end_of_integer_part = p; - int64_t digit_count = int64_t(end_of_integer_part - start_digits); - if (store_spans) { - answer.integer = span(start_digits, size_t(digit_count)); - } - SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(basic_json_fmt) { - // at least 1 digit in integer part, without leading zeros - if (digit_count == 0) { - return report_parse_error(p, parse_error::no_digits_in_integer_part); - } - if ((start_digits[0] == UC('0') && digit_count > 1)) { - return report_parse_error(start_digits, - parse_error::leading_zeros_in_integer_part); - } - } - - int64_t exponent = 0; - bool const has_decimal_point = (p != pend) && (*p == decimal_point); - if (has_decimal_point) { - ++p; - UC const *before = p; - // can occur at most twice without overflowing, but let it occur more, since - // for integers with many digits, digit parsing is the primary bottleneck. - loop_parse_if_eight_digits(p, pend, i); - - while ((p != pend) && is_integer(*p)) { - uint8_t digit = uint8_t(*p - UC('0')); - ++p; - i = i * 10 + digit; // in rare cases, this will overflow, but that's ok - } - exponent = before - p; - if (store_spans) { - answer.fraction = span(before, size_t(p - before)); - } - digit_count -= exponent; - } - SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(basic_json_fmt) { - // at least 1 digit in fractional part - if (has_decimal_point && exponent == 0) { - return report_parse_error(p, - parse_error::no_digits_in_fractional_part); - } - } - else if (digit_count == 0) { // we must have encountered at least one integer! - return report_parse_error(p, parse_error::no_digits_in_mantissa); - } - int64_t exp_number = 0; // explicit exponential part - if ((uint64_t(fmt & chars_format::scientific) && (p != pend) && - ((UC('e') == *p) || (UC('E') == *p))) || - (uint64_t(fmt & detail::basic_fortran_fmt) && (p != pend) && - ((UC('+') == *p) || (UC('-') == *p) || (UC('d') == *p) || - (UC('D') == *p)))) { - UC const *location_of_e = p; - if ((UC('e') == *p) || (UC('E') == *p) || (UC('d') == *p) || - (UC('D') == *p)) { - ++p; - } - bool neg_exp = false; - if ((p != pend) && (UC('-') == *p)) { - neg_exp = true; - ++p; - } else if ((p != pend) && - (UC('+') == - *p)) { // '+' on exponent is allowed by C++17 20.19.3.(7.1) - ++p; - } - if ((p == pend) || !is_integer(*p)) { - if (!uint64_t(fmt & chars_format::fixed)) { - // The exponential part is invalid for scientific notation, so it must - // be a trailing token for fixed notation. However, fixed notation is - // disabled, so report a scientific notation error. - return report_parse_error(p, parse_error::missing_exponential_part); - } - // Otherwise, we will be ignoring the 'e'. - p = location_of_e; - } else { - while ((p != pend) && is_integer(*p)) { - uint8_t digit = uint8_t(*p - UC('0')); - if (exp_number < 0x10000000) { - exp_number = 10 * exp_number + digit; - } - ++p; - } - if (neg_exp) { - exp_number = -exp_number; - } - exponent += exp_number; - } - } else { - // If it scientific and not fixed, we have to bail out. - if (uint64_t(fmt & chars_format::scientific) && - !uint64_t(fmt & chars_format::fixed)) { - return report_parse_error(p, parse_error::missing_exponential_part); - } - } - answer.lastmatch = p; - answer.valid = true; - - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon. - // - // We can deal with up to 19 digits. - if (digit_count > 19) { // this is uncommon - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - // We need to be mindful of the case where we only have zeroes... - // E.g., 0.000000000...000. - UC const *start = start_digits; - while ((start != pend) && (*start == UC('0') || *start == decimal_point)) { - if (*start == UC('0')) { - digit_count--; - } - start++; - } - - if (digit_count > 19) { - answer.too_many_digits = true; - // The truncation recompute below reads the integer/fraction spans. When - // store_spans is false we didn't materialize them, so just flag - // too_many_digits; the caller re-parses with store_spans=true to obtain - // the corrected mantissa/exponent before taking the slow path. - if (store_spans) { - // Let us start again, this time, avoiding overflows. - // We don't need to call if is_integer, since we use the - // pre-tokenized spans from above. - i = 0; - p = answer.integer.ptr; - UC const *int_end = p + answer.integer.len(); - uint64_t const minimal_nineteen_digit_integer{1000000000000000000}; - while ((i < minimal_nineteen_digit_integer) && (p != int_end)) { - i = i * 10 + uint64_t(*p - UC('0')); - ++p; - } - if (i >= minimal_nineteen_digit_integer) { // We have a big integer - exponent = end_of_integer_part - p + exp_number; - } else { // We have a value with a fractional component. - p = answer.fraction.ptr; - UC const *frac_end = p + answer.fraction.len(); - while ((i < minimal_nineteen_digit_integer) && (p != frac_end)) { - i = i * 10 + uint64_t(*p - UC('0')); - ++p; - } - exponent = answer.fraction.ptr - p + exp_number; - } - // We have now corrected both exponent and i, to a truncated value - } - } - } - answer.exponent = exponent; - answer.mantissa = i; - return answer; -} - -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t -parse_int_string(UC const *p, UC const *pend, T &value, - parse_options_t options) { - chars_format const fmt = detail::adjust_for_feature_macros(options.format); - int const base = options.base; - - from_chars_result_t answer; - - UC const *const first = p; - - bool const negative = (*p == UC('-')); -#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO -#pragma warning(push) -#pragma warning(disable : 4127) -#endif - if (!std::is_signed::value && negative) { -#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO -#pragma warning(pop) -#endif - answer.ec = std::errc::invalid_argument; - answer.ptr = first; - return answer; - } - if ((*p == UC('-')) || - (uint64_t(fmt & chars_format::allow_leading_plus) && (*p == UC('+')))) { - ++p; - } - - UC const *const start_num = p; - - while (p != pend && *p == UC('0')) { - ++p; - } - - bool const has_leading_zeros = p > start_num; - - UC const *const start_digits = p; - - SIMDJSON_FASTFLOAT_IF_CONSTEXPR17( - (std::is_same::value && sizeof(UC) == 1)) { - if (base == 10) { - const size_t len = static_cast(pend - p); - if (len == 0) { - if (has_leading_zeros) { - value = 0; - answer.ec = std::errc(); - answer.ptr = p; - } else { - answer.ec = std::errc::invalid_argument; - answer.ptr = first; - } - return answer; - } - - uint32_t digits; - -#if SIMDJSON_FASTFLOAT_HAS_IS_CONSTANT_EVALUATED && SIMDJSON_FASTFLOAT_HAS_BIT_CAST - if (std::is_constant_evaluated()) { - uint8_t str[4]{}; - for (size_t j = 0; j < 4 && j < len; ++j) { - str[j] = static_cast(p[j]); - } - digits = std::bit_cast(str); -#if SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN - digits = byteswap_32(digits); -#endif - } -#else - if (false) { - } -#endif - else if (len >= 4) { - ::memcpy(&digits, p, 4); -#if SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN - digits = byteswap_32(digits); -#endif - } else { - uint32_t b0 = static_cast(p[0]); - uint32_t b1 = (len > 1) ? static_cast(p[1]) : 0xFFu; - uint32_t b2 = (len > 2) ? static_cast(p[2]) : 0xFFu; - uint32_t b3 = 0xFFu; - digits = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24); - } - - uint32_t magic = - ((digits + 0x46464646u) | (digits - 0x30303030u)) & 0x80808080u; - uint32_t tz = - static_cast(countr_zero_32(magic)); // 7, 15, 23, 31, or 32 - uint32_t nd = (tz == 32) ? 4 : (tz >> 3); - nd = static_cast(nd < len ? nd : len); - if (nd == 0) { - if (has_leading_zeros) { - value = 0; - answer.ec = std::errc(); - answer.ptr = p; - return answer; - } - answer.ec = std::errc::invalid_argument; - answer.ptr = first; - return answer; - } - if (nd > 3) { - const UC *q = p + nd; - size_t rem = len - nd; - while (rem) { - if (*q < UC('0') || *q > UC('9')) - break; - ++q; - --rem; - } - answer.ec = std::errc::result_out_of_range; - answer.ptr = q; - return answer; - } - - digits ^= 0x30303030u; - digits <<= ((4 - nd) * 8); - - uint32_t check = ((digits >> 24) & 0xff) | ((digits >> 8) & 0xff00) | - ((digits << 8) & 0xff0000); - if (check > 0x00020505) { - answer.ec = std::errc::result_out_of_range; - answer.ptr = p + nd; - return answer; - } - value = static_cast((0x640a01 * digits) >> 24); - answer.ec = std::errc(); - answer.ptr = p + nd; - return answer; - } - } - - SIMDJSON_FASTFLOAT_IF_CONSTEXPR17( - (std::is_same::value && sizeof(UC) == 1)) { - if (base == 10) { - const size_t len = size_t(pend - p); - if (len == 0) { - if (has_leading_zeros) { - value = 0; - answer.ec = std::errc(); - answer.ptr = p; - } else { - answer.ec = std::errc::invalid_argument; - answer.ptr = first; - } - return answer; - } - - if (len >= 4) { - uint32_t digits = read4_to_u32(p); - if (is_made_of_four_digits_fast(digits)) { - uint32_t v = parse_four_digits_unrolled(digits); - if (len >= 5 && is_integer(p[4])) { - v = v * 10 + uint32_t(p[4] - '0'); - if (len >= 6 && is_integer(p[5])) { - answer.ec = std::errc::result_out_of_range; - const UC *q = p + 5; - while (q != pend && is_integer(*q)) { - q++; - } - answer.ptr = q; - return answer; - } - if (v > 65535) { - answer.ec = std::errc::result_out_of_range; - answer.ptr = p + 5; - return answer; - } - value = uint16_t(v); - answer.ec = std::errc(); - answer.ptr = p + 5; - return answer; - } - // 4 digits - value = uint16_t(v); - answer.ec = std::errc(); - answer.ptr = p + 4; - return answer; - } - } - } - } - - uint64_t i = 0; - if (base == 10) { - loop_parse_if_eight_digits(p, pend, i); // use SIMD if possible - } - while (p != pend) { - uint8_t digit = ch_to_digit(*p); - if (digit >= base) { - break; - } - i = uint64_t(base) * i + digit; // might overflow, check this later - p++; - } - - size_t digit_count = size_t(p - start_digits); - - if (digit_count == 0) { - if (has_leading_zeros) { - value = 0; - answer.ec = std::errc(); - answer.ptr = p; - } else { - answer.ec = std::errc::invalid_argument; - answer.ptr = first; - } - return answer; - } - - answer.ptr = p; - - // check u64 overflow - size_t max_digits = max_digits_u64(base); - if (digit_count > max_digits) { - answer.ec = std::errc::result_out_of_range; - return answer; - } - // this check can be eliminated for all other types, but they will all require - // a max_digits(base) equivalent - if (digit_count == max_digits) { - // At the max_digits boundary the accumulator `i` may have wrapped around - // 2^64. A plain `i < min_safe_u64(base)` test is not sufficient: for any - // base whose max_digits-length range exceeds 2^64 (base 10 reaches - // ~5.4 * 2^64 at 20 digits) the value can wrap a whole multiple of 2^64 and - // land back above min_safe, slipping through. Decide exactly in O(1) using - // the leading digit, following the approach used in simdjson: - // ms == min_safe_u64(base) == base^(max_digits-1), the smallest - // max_digits-length value. - // dmax == the largest leading digit whose number can still fit in u64. - // The leading-digit band [d*ms, (d+1)*ms) has width ms < 2^64, so within - // the single band where d == dmax the value straddles 2^64 at most once, - // and a single threshold separates wrapped from non-wrapped values. A - // leading digit above dmax always overflows; below dmax always fits. - uint64_t const ms = min_safe_u64(base); - uint64_t const dmax = (std::numeric_limits::max)() / ms; - uint64_t const lead = ch_to_digit(*start_digits); - if (lead > dmax || (lead == dmax && i < dmax * ms)) { - answer.ec = std::errc::result_out_of_range; - return answer; - } - } - - // check other types overflow - if (!std::is_same::value) { - if (i > uint64_t((std::numeric_limits::max)()) + uint64_t(negative)) { - answer.ec = std::errc::result_out_of_range; - return answer; - } - } - - if (negative) { -#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO -#pragma warning(push) -#pragma warning(disable : 4146) -#endif - // this weird workaround is required because: - // - converting unsigned to signed when its value is greater than signed max - // is UB pre-C++23. - // - reinterpret_casting (~i + 1) would work, but it is not constexpr - // this is always optimized into a neg instruction (note: T is an integer - // type) - value = T(-(std::numeric_limits::max)() - - T(i - uint64_t((std::numeric_limits::max)()))); -#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO -#pragma warning(pop) -#endif - } else { - value = T(i); - } - - answer.ec = std::errc(); - return answer; -} - -} // namespace simdjson_fast_float - -#endif - -#ifndef SIMDJSON_FASTFLOAT_FAST_TABLE_H -#define SIMDJSON_FASTFLOAT_FAST_TABLE_H - -#include - -namespace simdjson_fast_float { - -/** - * When mapping numbers from decimal to binary, - * we go from w * 10^q to m * 2^p but we have - * 10^q = 5^q * 2^q, so effectively - * we are trying to match - * w * 2^q * 5^q to m * 2^p. Thus the powers of two - * are not a concern since they can be represented - * exactly using the binary notation, only the powers of five - * affect the binary significand. - */ - -/** - * The smallest non-zero float (binary64) is 2^-1074. - * We take as input numbers of the form w x 10^q where w < 2^64. - * We have that w * 10^-343 < 2^(64-344) 5^-343 < 2^-1076. - * However, we have that - * (2^64-1) * 10^-342 = (2^64-1) * 2^-342 * 5^-342 > 2^-1074. - * Thus it is possible for a number of the form w * 10^-342 where - * w is a 64-bit value to be a non-zero floating-point number. - ********* - * Any number of form w * 10^309 where w>= 1 is going to be - * infinite in binary64 so we never need to worry about powers - * of 5 greater than 308. - */ -template struct powers_template { - - constexpr static int smallest_power_of_five = - binary_format::smallest_power_of_ten(); - constexpr static int largest_power_of_five = - binary_format::largest_power_of_ten(); - constexpr static int number_of_entries = - 2 * (largest_power_of_five - smallest_power_of_five + 1); - // Powers of five from 5^-342 all the way to 5^308 rounded toward one. - constexpr static uint64_t power_of_five_128[number_of_entries] = { - 0xeef453d6923bd65a, 0x113faa2906a13b3f, - 0x9558b4661b6565f8, 0x4ac7ca59a424c507, - 0xbaaee17fa23ebf76, 0x5d79bcf00d2df649, - 0xe95a99df8ace6f53, 0xf4d82c2c107973dc, - 0x91d8a02bb6c10594, 0x79071b9b8a4be869, - 0xb64ec836a47146f9, 0x9748e2826cdee284, - 0xe3e27a444d8d98b7, 0xfd1b1b2308169b25, - 0x8e6d8c6ab0787f72, 0xfe30f0f5e50e20f7, - 0xb208ef855c969f4f, 0xbdbd2d335e51a935, - 0xde8b2b66b3bc4723, 0xad2c788035e61382, - 0x8b16fb203055ac76, 0x4c3bcb5021afcc31, - 0xaddcb9e83c6b1793, 0xdf4abe242a1bbf3d, - 0xd953e8624b85dd78, 0xd71d6dad34a2af0d, - 0x87d4713d6f33aa6b, 0x8672648c40e5ad68, - 0xa9c98d8ccb009506, 0x680efdaf511f18c2, - 0xd43bf0effdc0ba48, 0x212bd1b2566def2, - 0x84a57695fe98746d, 0x14bb630f7604b57, - 0xa5ced43b7e3e9188, 0x419ea3bd35385e2d, - 0xcf42894a5dce35ea, 0x52064cac828675b9, - 0x818995ce7aa0e1b2, 0x7343efebd1940993, - 0xa1ebfb4219491a1f, 0x1014ebe6c5f90bf8, - 0xca66fa129f9b60a6, 0xd41a26e077774ef6, - 0xfd00b897478238d0, 0x8920b098955522b4, - 0x9e20735e8cb16382, 0x55b46e5f5d5535b0, - 0xc5a890362fddbc62, 0xeb2189f734aa831d, - 0xf712b443bbd52b7b, 0xa5e9ec7501d523e4, - 0x9a6bb0aa55653b2d, 0x47b233c92125366e, - 0xc1069cd4eabe89f8, 0x999ec0bb696e840a, - 0xf148440a256e2c76, 0xc00670ea43ca250d, - 0x96cd2a865764dbca, 0x380406926a5e5728, - 0xbc807527ed3e12bc, 0xc605083704f5ecf2, - 0xeba09271e88d976b, 0xf7864a44c633682e, - 0x93445b8731587ea3, 0x7ab3ee6afbe0211d, - 0xb8157268fdae9e4c, 0x5960ea05bad82964, - 0xe61acf033d1a45df, 0x6fb92487298e33bd, - 0x8fd0c16206306bab, 0xa5d3b6d479f8e056, - 0xb3c4f1ba87bc8696, 0x8f48a4899877186c, - 0xe0b62e2929aba83c, 0x331acdabfe94de87, - 0x8c71dcd9ba0b4925, 0x9ff0c08b7f1d0b14, - 0xaf8e5410288e1b6f, 0x7ecf0ae5ee44dd9, - 0xdb71e91432b1a24a, 0xc9e82cd9f69d6150, - 0x892731ac9faf056e, 0xbe311c083a225cd2, - 0xab70fe17c79ac6ca, 0x6dbd630a48aaf406, - 0xd64d3d9db981787d, 0x92cbbccdad5b108, - 0x85f0468293f0eb4e, 0x25bbf56008c58ea5, - 0xa76c582338ed2621, 0xaf2af2b80af6f24e, - 0xd1476e2c07286faa, 0x1af5af660db4aee1, - 0x82cca4db847945ca, 0x50d98d9fc890ed4d, - 0xa37fce126597973c, 0xe50ff107bab528a0, - 0xcc5fc196fefd7d0c, 0x1e53ed49a96272c8, - 0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7a, - 0x9faacf3df73609b1, 0x77b191618c54e9ac, - 0xc795830d75038c1d, 0xd59df5b9ef6a2417, - 0xf97ae3d0d2446f25, 0x4b0573286b44ad1d, - 0x9becce62836ac577, 0x4ee367f9430aec32, - 0xc2e801fb244576d5, 0x229c41f793cda73f, - 0xf3a20279ed56d48a, 0x6b43527578c1110f, - 0x9845418c345644d6, 0x830a13896b78aaa9, - 0xbe5691ef416bd60c, 0x23cc986bc656d553, - 0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa8, - 0x94b3a202eb1c3f39, 0x7bf7d71432f3d6a9, - 0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc53, - 0xe858ad248f5c22c9, 0xd1b3400f8f9cff68, - 0x91376c36d99995be, 0x23100809b9c21fa1, - 0xb58547448ffffb2d, 0xabd40a0c2832a78a, - 0xe2e69915b3fff9f9, 0x16c90c8f323f516c, - 0x8dd01fad907ffc3b, 0xae3da7d97f6792e3, - 0xb1442798f49ffb4a, 0x99cd11cfdf41779c, - 0xdd95317f31c7fa1d, 0x40405643d711d583, - 0x8a7d3eef7f1cfc52, 0x482835ea666b2572, - 0xad1c8eab5ee43b66, 0xda3243650005eecf, - 0xd863b256369d4a40, 0x90bed43e40076a82, - 0x873e4f75e2224e68, 0x5a7744a6e804a291, - 0xa90de3535aaae202, 0x711515d0a205cb36, - 0xd3515c2831559a83, 0xd5a5b44ca873e03, - 0x8412d9991ed58091, 0xe858790afe9486c2, - 0xa5178fff668ae0b6, 0x626e974dbe39a872, - 0xce5d73ff402d98e3, 0xfb0a3d212dc8128f, - 0x80fa687f881c7f8e, 0x7ce66634bc9d0b99, - 0xa139029f6a239f72, 0x1c1fffc1ebc44e80, - 0xc987434744ac874e, 0xa327ffb266b56220, - 0xfbe9141915d7a922, 0x4bf1ff9f0062baa8, - 0x9d71ac8fada6c9b5, 0x6f773fc3603db4a9, - 0xc4ce17b399107c22, 0xcb550fb4384d21d3, - 0xf6019da07f549b2b, 0x7e2a53a146606a48, - 0x99c102844f94e0fb, 0x2eda7444cbfc426d, - 0xc0314325637a1939, 0xfa911155fefb5308, - 0xf03d93eebc589f88, 0x793555ab7eba27ca, - 0x96267c7535b763b5, 0x4bc1558b2f3458de, - 0xbbb01b9283253ca2, 0x9eb1aaedfb016f16, - 0xea9c227723ee8bcb, 0x465e15a979c1cadc, - 0x92a1958a7675175f, 0xbfacd89ec191ec9, - 0xb749faed14125d36, 0xcef980ec671f667b, - 0xe51c79a85916f484, 0x82b7e12780e7401a, - 0x8f31cc0937ae58d2, 0xd1b2ecb8b0908810, - 0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa15, - 0xdfbdcece67006ac9, 0x67a791e093e1d49a, - 0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e0, - 0xaecc49914078536d, 0x58fae9f773886e18, - 0xda7f5bf590966848, 0xaf39a475506a899e, - 0x888f99797a5e012d, 0x6d8406c952429603, - 0xaab37fd7d8f58178, 0xc8e5087ba6d33b83, - 0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a64, - 0x855c3be0a17fcd26, 0x5cf2eea09a55067f, - 0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481e, - 0xd0601d8efc57b08b, 0xf13b94daf124da26, - 0x823c12795db6ce57, 0x76c53d08d6b70858, - 0xa2cb1717b52481ed, 0x54768c4b0c64ca6e, - 0xcb7ddcdda26da268, 0xa9942f5dcf7dfd09, - 0xfe5d54150b090b02, 0xd3f93b35435d7c4c, - 0x9efa548d26e5a6e1, 0xc47bc5014a1a6daf, - 0xc6b8e9b0709f109a, 0x359ab6419ca1091b, - 0xf867241c8cc6d4c0, 0xc30163d203c94b62, - 0x9b407691d7fc44f8, 0x79e0de63425dcf1d, - 0xc21094364dfb5636, 0x985915fc12f542e4, - 0xf294b943e17a2bc4, 0x3e6f5b7b17b2939d, - 0x979cf3ca6cec5b5a, 0xa705992ceecf9c42, - 0xbd8430bd08277231, 0x50c6ff782a838353, - 0xece53cec4a314ebd, 0xa4f8bf5635246428, - 0x940f4613ae5ed136, 0x871b7795e136be99, - 0xb913179899f68584, 0x28e2557b59846e3f, - 0xe757dd7ec07426e5, 0x331aeada2fe589cf, - 0x9096ea6f3848984f, 0x3ff0d2c85def7621, - 0xb4bca50b065abe63, 0xfed077a756b53a9, - 0xe1ebce4dc7f16dfb, 0xd3e8495912c62894, - 0x8d3360f09cf6e4bd, 0x64712dd7abbbd95c, - 0xb080392cc4349dec, 0xbd8d794d96aacfb3, - 0xdca04777f541c567, 0xecf0d7a0fc5583a0, - 0x89e42caaf9491b60, 0xf41686c49db57244, - 0xac5d37d5b79b6239, 0x311c2875c522ced5, - 0xd77485cb25823ac7, 0x7d633293366b828b, - 0x86a8d39ef77164bc, 0xae5dff9c02033197, - 0xa8530886b54dbdeb, 0xd9f57f830283fdfc, - 0xd267caa862a12d66, 0xd072df63c324fd7b, - 0x8380dea93da4bc60, 0x4247cb9e59f71e6d, - 0xa46116538d0deb78, 0x52d9be85f074e608, - 0xcd795be870516656, 0x67902e276c921f8b, - 0x806bd9714632dff6, 0xba1cd8a3db53b6, - 0xa086cfcd97bf97f3, 0x80e8a40eccd228a4, - 0xc8a883c0fdaf7df0, 0x6122cd128006b2cd, - 0xfad2a4b13d1b5d6c, 0x796b805720085f81, - 0x9cc3a6eec6311a63, 0xcbe3303674053bb0, - 0xc3f490aa77bd60fc, 0xbedbfc4411068a9c, - 0xf4f1b4d515acb93b, 0xee92fb5515482d44, - 0x991711052d8bf3c5, 0x751bdd152d4d1c4a, - 0xbf5cd54678eef0b6, 0xd262d45a78a0635d, - 0xef340a98172aace4, 0x86fb897116c87c34, - 0x9580869f0e7aac0e, 0xd45d35e6ae3d4da0, - 0xbae0a846d2195712, 0x8974836059cca109, - 0xe998d258869facd7, 0x2bd1a438703fc94b, - 0x91ff83775423cc06, 0x7b6306a34627ddcf, - 0xb67f6455292cbf08, 0x1a3bc84c17b1d542, - 0xe41f3d6a7377eeca, 0x20caba5f1d9e4a93, - 0x8e938662882af53e, 0x547eb47b7282ee9c, - 0xb23867fb2a35b28d, 0xe99e619a4f23aa43, - 0xdec681f9f4c31f31, 0x6405fa00e2ec94d4, - 0x8b3c113c38f9f37e, 0xde83bc408dd3dd04, - 0xae0b158b4738705e, 0x9624ab50b148d445, - 0xd98ddaee19068c76, 0x3badd624dd9b0957, - 0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d6, - 0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4c, - 0xd47487cc8470652b, 0x7647c3200069671f, - 0x84c8d4dfd2c63f3b, 0x29ecd9f40041e073, - 0xa5fb0a17c777cf09, 0xf468107100525890, - 0xcf79cc9db955c2cc, 0x7182148d4066eeb4, - 0x81ac1fe293d599bf, 0xc6f14cd848405530, - 0xa21727db38cb002f, 0xb8ada00e5a506a7c, - 0xca9cf1d206fdc03b, 0xa6d90811f0e4851c, - 0xfd442e4688bd304a, 0x908f4a166d1da663, - 0x9e4a9cec15763e2e, 0x9a598e4e043287fe, - 0xc5dd44271ad3cdba, 0x40eff1e1853f29fd, - 0xf7549530e188c128, 0xd12bee59e68ef47c, - 0x9a94dd3e8cf578b9, 0x82bb74f8301958ce, - 0xc13a148e3032d6e7, 0xe36a52363c1faf01, - 0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac1, - 0x96f5600f15a7b7e5, 0x29ab103a5ef8c0b9, - 0xbcb2b812db11a5de, 0x7415d448f6b6f0e7, - 0xebdf661791d60f56, 0x111b495b3464ad21, - 0x936b9fcebb25c995, 0xcab10dd900beec34, - 0xb84687c269ef3bfb, 0x3d5d514f40eea742, - 0xe65829b3046b0afa, 0xcb4a5a3112a5112, - 0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ab, - 0xb3f4e093db73a093, 0x59ed216765690f56, - 0xe0f218b8d25088b8, 0x306869c13ec3532c, - 0x8c974f7383725573, 0x1e414218c73a13fb, - 0xafbd2350644eeacf, 0xe5d1929ef90898fa, - 0xdbac6c247d62a583, 0xdf45f746b74abf39, - 0x894bc396ce5da772, 0x6b8bba8c328eb783, - 0xab9eb47c81f5114f, 0x66ea92f3f326564, - 0xd686619ba27255a2, 0xc80a537b0efefebd, - 0x8613fd0145877585, 0xbd06742ce95f5f36, - 0xa798fc4196e952e7, 0x2c48113823b73704, - 0xd17f3b51fca3a7a0, 0xf75a15862ca504c5, - 0x82ef85133de648c4, 0x9a984d73dbe722fb, - 0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebba, - 0xcc963fee10b7d1b3, 0x318df905079926a8, - 0xffbbcfe994e5c61f, 0xfdf17746497f7052, - 0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa633, - 0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc0, - 0xf9bd690a1b68637b, 0x3dfdce7aa3c673b0, - 0x9c1661a651213e2d, 0x6bea10ca65c084e, - 0xc31bfa0fe5698db8, 0x486e494fcff30a62, - 0xf3e2f893dec3f126, 0x5a89dba3c3efccfa, - 0x986ddb5c6b3a76b7, 0xf89629465a75e01c, - 0xbe89523386091465, 0xf6bbb397f1135823, - 0xee2ba6c0678b597f, 0x746aa07ded582e2c, - 0x94db483840b717ef, 0xa8c2a44eb4571cdc, - 0xba121a4650e4ddeb, 0x92f34d62616ce413, - 0xe896a0d7e51e1566, 0x77b020baf9c81d17, - 0x915e2486ef32cd60, 0xace1474dc1d122e, - 0xb5b5ada8aaff80b8, 0xd819992132456ba, - 0xe3231912d5bf60e6, 0x10e1fff697ed6c69, - 0x8df5efabc5979c8f, 0xca8d3ffa1ef463c1, - 0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb2, - 0xddd0467c64bce4a0, 0xac7cb3f6d05ddbde, - 0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96b, - 0xad4ab7112eb3929d, 0x86c16c98d2c953c6, - 0xd89d64d57a607744, 0xe871c7bf077ba8b7, - 0x87625f056c7c4a8b, 0x11471cd764ad4972, - 0xa93af6c6c79b5d2d, 0xd598e40d3dd89bcf, - 0xd389b47879823479, 0x4aff1d108d4ec2c3, - 0x843610cb4bf160cb, 0xcedf722a585139ba, - 0xa54394fe1eedb8fe, 0xc2974eb4ee658828, - 0xce947a3da6a9273e, 0x733d226229feea32, - 0x811ccc668829b887, 0x806357d5a3f525f, - 0xa163ff802a3426a8, 0xca07c2dcb0cf26f7, - 0xc9bcff6034c13052, 0xfc89b393dd02f0b5, - 0xfc2c3f3841f17c67, 0xbbac2078d443ace2, - 0x9d9ba7832936edc0, 0xd54b944b84aa4c0d, - 0xc5029163f384a931, 0xa9e795e65d4df11, - 0xf64335bcf065d37d, 0x4d4617b5ff4a16d5, - 0x99ea0196163fa42e, 0x504bced1bf8e4e45, - 0xc06481fb9bcf8d39, 0xe45ec2862f71e1d6, - 0xf07da27a82c37088, 0x5d767327bb4e5a4c, - 0x964e858c91ba2655, 0x3a6a07f8d510f86f, - 0xbbe226efb628afea, 0x890489f70a55368b, - 0xeadab0aba3b2dbe5, 0x2b45ac74ccea842e, - 0x92c8ae6b464fc96f, 0x3b0b8bc90012929d, - 0xb77ada0617e3bbcb, 0x9ce6ebb40173744, - 0xe55990879ddcaabd, 0xcc420a6a101d0515, - 0x8f57fa54c2a9eab6, 0x9fa946824a12232d, - 0xb32df8e9f3546564, 0x47939822dc96abf9, - 0xdff9772470297ebd, 0x59787e2b93bc56f7, - 0x8bfbea76c619ef36, 0x57eb4edb3c55b65a, - 0xaefae51477a06b03, 0xede622920b6b23f1, - 0xdab99e59958885c4, 0xe95fab368e45eced, - 0x88b402f7fd75539b, 0x11dbcb0218ebb414, - 0xaae103b5fcd2a881, 0xd652bdc29f26a119, - 0xd59944a37c0752a2, 0x4be76d3346f0495f, - 0x857fcae62d8493a5, 0x6f70a4400c562ddb, - 0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb952, - 0xd097ad07a71f26b2, 0x7e2000a41346a7a7, - 0x825ecc24c873782f, 0x8ed400668c0c28c8, - 0xa2f67f2dfa90563b, 0x728900802f0f32fa, - 0xcbb41ef979346bca, 0x4f2b40a03ad2ffb9, - 0xfea126b7d78186bc, 0xe2f610c84987bfa8, - 0x9f24b832e6b0f436, 0xdd9ca7d2df4d7c9, - 0xc6ede63fa05d3143, 0x91503d1c79720dbb, - 0xf8a95fcf88747d94, 0x75a44c6397ce912a, - 0x9b69dbe1b548ce7c, 0xc986afbe3ee11aba, - 0xc24452da229b021b, 0xfbe85badce996168, - 0xf2d56790ab41c2a2, 0xfae27299423fb9c3, - 0x97c560ba6b0919a5, 0xdccd879fc967d41a, - 0xbdb6b8e905cb600f, 0x5400e987bbc1c920, - 0xed246723473e3813, 0x290123e9aab23b68, - 0x9436c0760c86e30b, 0xf9a0b6720aaf6521, - 0xb94470938fa89bce, 0xf808e40e8d5b3e69, - 0xe7958cb87392c2c2, 0xb60b1d1230b20e04, - 0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c2, - 0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af3, - 0xe2280b6c20dd5232, 0x25c6da63c38de1b0, - 0x8d590723948a535f, 0x579c487e5a38ad0e, - 0xb0af48ec79ace837, 0x2d835a9df0c6d851, - 0xdcdb1b2798182244, 0xf8e431456cf88e65, - 0x8a08f0f8bf0f156b, 0x1b8e9ecb641b58ff, - 0xac8b2d36eed2dac5, 0xe272467e3d222f3f, - 0xd7adf884aa879177, 0x5b0ed81dcc6abb0f, - 0x86ccbb52ea94baea, 0x98e947129fc2b4e9, - 0xa87fea27a539e9a5, 0x3f2398d747b36224, - 0xd29fe4b18e88640e, 0x8eec7f0d19a03aad, - 0x83a3eeeef9153e89, 0x1953cf68300424ac, - 0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd7, - 0xcdb02555653131b6, 0x3792f412cb06794d, - 0x808e17555f3ebf11, 0xe2bbd88bbee40bd0, - 0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec4, - 0xc8de047564d20a8b, 0xf245825a5a445275, - 0xfb158592be068d2e, 0xeed6e2f0f0d56712, - 0x9ced737bb6c4183d, 0x55464dd69685606b, - 0xc428d05aa4751e4c, 0xaa97e14c3c26b886, - 0xf53304714d9265df, 0xd53dd99f4b3066a8, - 0x993fe2c6d07b7fab, 0xe546a8038efe4029, - 0xbf8fdb78849a5f96, 0xde98520472bdd033, - 0xef73d256a5c0f77c, 0x963e66858f6d4440, - 0x95a8637627989aad, 0xdde7001379a44aa8, - 0xbb127c53b17ec159, 0x5560c018580d5d52, - 0xe9d71b689dde71af, 0xaab8f01e6e10b4a6, - 0x9226712162ab070d, 0xcab3961304ca70e8, - 0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d22, - 0xe45c10c42a2b3b05, 0x8cb89a7db77c506a, - 0x8eb98a7a9a5b04e3, 0x77f3608e92adb242, - 0xb267ed1940f1c61c, 0x55f038b237591ed3, - 0xdf01e85f912e37a3, 0x6b6c46dec52f6688, - 0x8b61313bbabce2c6, 0x2323ac4b3b3da015, - 0xae397d8aa96c1b77, 0xabec975e0a0d081a, - 0xd9c7dced53c72255, 0x96e7bd358c904a21, - 0x881cea14545c7575, 0x7e50d64177da2e54, - 0xaa242499697392d2, 0xdde50bd1d5d0b9e9, - 0xd4ad2dbfc3d07787, 0x955e4ec64b44e864, - 0x84ec3c97da624ab4, 0xbd5af13bef0b113e, - 0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58e, - 0xcfb11ead453994ba, 0x67de18eda5814af2, - 0x81ceb32c4b43fcf4, 0x80eacf948770ced7, - 0xa2425ff75e14fc31, 0xa1258379a94d028d, - 0xcad2f7f5359a3b3e, 0x96ee45813a04330, - 0xfd87b5f28300ca0d, 0x8bca9d6e188853fc, - 0x9e74d1b791e07e48, 0x775ea264cf55347e, - 0xc612062576589dda, 0x95364afe032a819e, - 0xf79687aed3eec551, 0x3a83ddbd83f52205, - 0x9abe14cd44753b52, 0xc4926a9672793543, - 0xc16d9a0095928a27, 0x75b7053c0f178294, - 0xf1c90080baf72cb1, 0x5324c68b12dd6339, - 0x971da05074da7bee, 0xd3f6fc16ebca5e04, - 0xbce5086492111aea, 0x88f4bb1ca6bcf585, - 0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6, - 0x9392ee8e921d5d07, 0x3aff322e62439fd0, - 0xb877aa3236a4b449, 0x9befeb9fad487c3, - 0xe69594bec44de15b, 0x4c2ebe687989a9b4, - 0x901d7cf73ab0acd9, 0xf9d37014bf60a11, - 0xb424dc35095cd80f, 0x538484c19ef38c95, - 0xe12e13424bb40e13, 0x2865a5f206b06fba, - 0x8cbccc096f5088cb, 0xf93f87b7442e45d4, - 0xafebff0bcb24aafe, 0xf78f69a51539d749, - 0xdbe6fecebdedd5be, 0xb573440e5a884d1c, - 0x89705f4136b4a597, 0x31680a88f8953031, - 0xabcc77118461cefc, 0xfdc20d2b36ba7c3e, - 0xd6bf94d5e57a42bc, 0x3d32907604691b4d, - 0x8637bd05af6c69b5, 0xa63f9a49c2c1b110, - 0xa7c5ac471b478423, 0xfcf80dc33721d54, - 0xd1b71758e219652b, 0xd3c36113404ea4a9, - 0x83126e978d4fdf3b, 0x645a1cac083126ea, - 0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4, - 0xcccccccccccccccc, 0xcccccccccccccccd, - 0x8000000000000000, 0x0, - 0xa000000000000000, 0x0, - 0xc800000000000000, 0x0, - 0xfa00000000000000, 0x0, - 0x9c40000000000000, 0x0, - 0xc350000000000000, 0x0, - 0xf424000000000000, 0x0, - 0x9896800000000000, 0x0, - 0xbebc200000000000, 0x0, - 0xee6b280000000000, 0x0, - 0x9502f90000000000, 0x0, - 0xba43b74000000000, 0x0, - 0xe8d4a51000000000, 0x0, - 0x9184e72a00000000, 0x0, - 0xb5e620f480000000, 0x0, - 0xe35fa931a0000000, 0x0, - 0x8e1bc9bf04000000, 0x0, - 0xb1a2bc2ec5000000, 0x0, - 0xde0b6b3a76400000, 0x0, - 0x8ac7230489e80000, 0x0, - 0xad78ebc5ac620000, 0x0, - 0xd8d726b7177a8000, 0x0, - 0x878678326eac9000, 0x0, - 0xa968163f0a57b400, 0x0, - 0xd3c21bcecceda100, 0x0, - 0x84595161401484a0, 0x0, - 0xa56fa5b99019a5c8, 0x0, - 0xcecb8f27f4200f3a, 0x0, - 0x813f3978f8940984, 0x4000000000000000, - 0xa18f07d736b90be5, 0x5000000000000000, - 0xc9f2c9cd04674ede, 0xa400000000000000, - 0xfc6f7c4045812296, 0x4d00000000000000, - 0x9dc5ada82b70b59d, 0xf020000000000000, - 0xc5371912364ce305, 0x6c28000000000000, - 0xf684df56c3e01bc6, 0xc732000000000000, - 0x9a130b963a6c115c, 0x3c7f400000000000, - 0xc097ce7bc90715b3, 0x4b9f100000000000, - 0xf0bdc21abb48db20, 0x1e86d40000000000, - 0x96769950b50d88f4, 0x1314448000000000, - 0xbc143fa4e250eb31, 0x17d955a000000000, - 0xeb194f8e1ae525fd, 0x5dcfab0800000000, - 0x92efd1b8d0cf37be, 0x5aa1cae500000000, - 0xb7abc627050305ad, 0xf14a3d9e40000000, - 0xe596b7b0c643c719, 0x6d9ccd05d0000000, - 0x8f7e32ce7bea5c6f, 0xe4820023a2000000, - 0xb35dbf821ae4f38b, 0xdda2802c8a800000, - 0xe0352f62a19e306e, 0xd50b2037ad200000, - 0x8c213d9da502de45, 0x4526f422cc340000, - 0xaf298d050e4395d6, 0x9670b12b7f410000, - 0xdaf3f04651d47b4c, 0x3c0cdd765f114000, - 0x88d8762bf324cd0f, 0xa5880a69fb6ac800, - 0xab0e93b6efee0053, 0x8eea0d047a457a00, - 0xd5d238a4abe98068, 0x72a4904598d6d880, - 0x85a36366eb71f041, 0x47a6da2b7f864750, - 0xa70c3c40a64e6c51, 0x999090b65f67d924, - 0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d, - 0x82818f1281ed449f, 0xbff8f10e7a8921a4, - 0xa321f2d7226895c7, 0xaff72d52192b6a0d, - 0xcbea6f8ceb02bb39, 0x9bf4f8a69f764490, - 0xfee50b7025c36a08, 0x2f236d04753d5b4, - 0x9f4f2726179a2245, 0x1d762422c946590, - 0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef5, - 0xf8ebad2b84e0d58b, 0xd2e0898765a7deb2, - 0x9b934c3b330c8577, 0x63cc55f49f88eb2f, - 0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fb, - 0xf316271c7fc3908a, 0x8bef464e3945ef7a, - 0x97edd871cfda3a56, 0x97758bf0e3cbb5ac, - 0xbde94e8e43d0c8ec, 0x3d52eeed1cbea317, - 0xed63a231d4c4fb27, 0x4ca7aaa863ee4bdd, - 0x945e455f24fb1cf8, 0x8fe8caa93e74ef6a, - 0xb975d6b6ee39e436, 0xb3e2fd538e122b44, - 0xe7d34c64a9c85d44, 0x60dbbca87196b616, - 0x90e40fbeea1d3a4a, 0xbc8955e946fe31cd, - 0xb51d13aea4a488dd, 0x6babab6398bdbe41, - 0xe264589a4dcdab14, 0xc696963c7eed2dd1, - 0x8d7eb76070a08aec, 0xfc1e1de5cf543ca2, - 0xb0de65388cc8ada8, 0x3b25a55f43294bcb, - 0xdd15fe86affad912, 0x49ef0eb713f39ebe, - 0x8a2dbf142dfcc7ab, 0x6e3569326c784337, - 0xacb92ed9397bf996, 0x49c2c37f07965404, - 0xd7e77a8f87daf7fb, 0xdc33745ec97be906, - 0x86f0ac99b4e8dafd, 0x69a028bb3ded71a3, - 0xa8acd7c0222311bc, 0xc40832ea0d68ce0c, - 0xd2d80db02aabd62b, 0xf50a3fa490c30190, - 0x83c7088e1aab65db, 0x792667c6da79e0fa, - 0xa4b8cab1a1563f52, 0x577001b891185938, - 0xcde6fd5e09abcf26, 0xed4c0226b55e6f86, - 0x80b05e5ac60b6178, 0x544f8158315b05b4, - 0xa0dc75f1778e39d6, 0x696361ae3db1c721, - 0xc913936dd571c84c, 0x3bc3a19cd1e38e9, - 0xfb5878494ace3a5f, 0x4ab48a04065c723, - 0x9d174b2dcec0e47b, 0x62eb0d64283f9c76, - 0xc45d1df942711d9a, 0x3ba5d0bd324f8394, - 0xf5746577930d6500, 0xca8f44ec7ee36479, - 0x9968bf6abbe85f20, 0x7e998b13cf4e1ecb, - 0xbfc2ef456ae276e8, 0x9e3fedd8c321a67e, - 0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101e, - 0x95d04aee3b80ece5, 0xbba1f1d158724a12, - 0xbb445da9ca61281f, 0x2a8a6e45ae8edc97, - 0xea1575143cf97226, 0xf52d09d71a3293bd, - 0x924d692ca61be758, 0x593c2626705f9c56, - 0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836c, - 0xe498f455c38b997a, 0xb6dfb9c0f956447, - 0x8edf98b59a373fec, 0x4724bd4189bd5eac, - 0xb2977ee300c50fe7, 0x58edec91ec2cb657, - 0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ed, - 0x8b865b215899f46c, 0xbd79e0d20082ee74, - 0xae67f1e9aec07187, 0xecd8590680a3aa11, - 0xda01ee641a708de9, 0xe80e6f4820cc9495, - 0x884134fe908658b2, 0x3109058d147fdcdd, - 0xaa51823e34a7eede, 0xbd4b46f0599fd415, - 0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91a, - 0x850fadc09923329e, 0x3e2cf6bc604ddb0, - 0xa6539930bf6bff45, 0x84db8346b786151c, - 0xcfe87f7cef46ff16, 0xe612641865679a63, - 0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07e, - 0xa26da3999aef7749, 0xe3be5e330f38f09d, - 0xcb090c8001ab551c, 0x5cadf5bfd3072cc5, - 0xfdcb4fa002162a63, 0x73d9732fc7c8f7f6, - 0x9e9f11c4014dda7e, 0x2867e7fddcdd9afa, - 0xc646d63501a1511d, 0xb281e1fd541501b8, - 0xf7d88bc24209a565, 0x1f225a7ca91a4226, - 0x9ae757596946075f, 0x3375788de9b06958, - 0xc1a12d2fc3978937, 0x52d6b1641c83ae, - 0xf209787bb47d6b84, 0xc0678c5dbd23a49a, - 0x9745eb4d50ce6332, 0xf840b7ba963646e0, - 0xbd176620a501fbff, 0xb650e5a93bc3d898, - 0xec5d3fa8ce427aff, 0xa3e51f138ab4cebe, - 0x93ba47c980e98cdf, 0xc66f336c36b10137, - 0xb8a8d9bbe123f017, 0xb80b0047445d4184, - 0xe6d3102ad96cec1d, 0xa60dc059157491e5, - 0x9043ea1ac7e41392, 0x87c89837ad68db2f, - 0xb454e4a179dd1877, 0x29babe4598c311fb, - 0xe16a1dc9d8545e94, 0xf4296dd6fef3d67a, - 0x8ce2529e2734bb1d, 0x1899e4a65f58660c, - 0xb01ae745b101e9e4, 0x5ec05dcff72e7f8f, - 0xdc21a1171d42645d, 0x76707543f4fa1f73, - 0x899504ae72497eba, 0x6a06494a791c53a8, - 0xabfa45da0edbde69, 0x487db9d17636892, - 0xd6f8d7509292d603, 0x45a9d2845d3c42b6, - 0x865b86925b9bc5c2, 0xb8a2392ba45a9b2, - 0xa7f26836f282b732, 0x8e6cac7768d7141e, - 0xd1ef0244af2364ff, 0x3207d795430cd926, - 0x8335616aed761f1f, 0x7f44e6bd49e807b8, - 0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a6, - 0xcd036837130890a1, 0x36dba887c37a8c0f, - 0x802221226be55a64, 0xc2494954da2c9789, - 0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6c, - 0xc83553c5c8965d3d, 0x6f92829494e5acc7, - 0xfa42a8b73abbf48c, 0xcb772339ba1f17f9, - 0x9c69a97284b578d7, 0xff2a760414536efb, - 0xc38413cf25e2d70d, 0xfef5138519684aba, - 0xf46518c2ef5b8cd1, 0x7eb258665fc25d69, - 0x98bf2f79d5993802, 0xef2f773ffbd97a61, - 0xbeeefb584aff8603, 0xaafb550ffacfd8fa, - 0xeeaaba2e5dbf6784, 0x95ba2a53f983cf38, - 0x952ab45cfa97a0b2, 0xdd945a747bf26183, - 0xba756174393d88df, 0x94f971119aeef9e4, - 0xe912b9d1478ceb17, 0x7a37cd5601aab85d, - 0x91abb422ccb812ee, 0xac62e055c10ab33a, - 0xb616a12b7fe617aa, 0x577b986b314d6009, - 0xe39c49765fdf9d94, 0xed5a7e85fda0b80b, - 0x8e41ade9fbebc27d, 0x14588f13be847307, - 0xb1d219647ae6b31c, 0x596eb2d8ae258fc8, - 0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bb, - 0x8aec23d680043bee, 0x25de7bb9480d5854, - 0xada72ccc20054ae9, 0xaf561aa79a10ae6a, - 0xd910f7ff28069da4, 0x1b2ba1518094da04, - 0x87aa9aff79042286, 0x90fb44d2f05d0842, - 0xa99541bf57452b28, 0x353a1607ac744a53, - 0xd3fa922f2d1675f2, 0x42889b8997915ce8, - 0x847c9b5d7c2e09b7, 0x69956135febada11, - 0xa59bc234db398c25, 0x43fab9837e699095, - 0xcf02b2c21207ef2e, 0x94f967e45e03f4bb, - 0x8161afb94b44f57d, 0x1d1be0eebac278f5, - 0xa1ba1ba79e1632dc, 0x6462d92a69731732, - 0xca28a291859bbf93, 0x7d7b8f7503cfdcfe, - 0xfcb2cb35e702af78, 0x5cda735244c3d43e, - 0x9defbf01b061adab, 0x3a0888136afa64a7, - 0xc56baec21c7a1916, 0x88aaa1845b8fdd0, - 0xf6c69a72a3989f5b, 0x8aad549e57273d45, - 0x9a3c2087a63f6399, 0x36ac54e2f678864b, - 0xc0cb28a98fcf3c7f, 0x84576a1bb416a7dd, - 0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d5, - 0x969eb7c47859e743, 0x9f644ae5a4b1b325, - 0xbc4665b596706114, 0x873d5d9f0dde1fee, - 0xeb57ff22fc0c7959, 0xa90cb506d155a7ea, - 0x9316ff75dd87cbd8, 0x9a7f12442d588f2, - 0xb7dcbf5354e9bece, 0xc11ed6d538aeb2f, - 0xe5d3ef282a242e81, 0x8f1668c8a86da5fa, - 0x8fa475791a569d10, 0xf96e017d694487bc, - 0xb38d92d760ec4455, 0x37c981dcc395a9ac, - 0xe070f78d3927556a, 0x85bbe253f47b1417, - 0x8c469ab843b89562, 0x93956d7478ccec8e, - 0xaf58416654a6babb, 0x387ac8d1970027b2, - 0xdb2e51bfe9d0696a, 0x6997b05fcc0319e, - 0x88fcf317f22241e2, 0x441fece3bdf81f03, - 0xab3c2fddeeaad25a, 0xd527e81cad7626c3, - 0xd60b3bd56a5586f1, 0x8a71e223d8d3b074, - 0x85c7056562757456, 0xf6872d5667844e49, - 0xa738c6bebb12d16c, 0xb428f8ac016561db, - 0xd106f86e69d785c7, 0xe13336d701beba52, - 0x82a45b450226b39c, 0xecc0024661173473, - 0xa34d721642b06084, 0x27f002d7f95d0190, - 0xcc20ce9bd35c78a5, 0x31ec038df7b441f4, - 0xff290242c83396ce, 0x7e67047175a15271, - 0x9f79a169bd203e41, 0xf0062c6e984d386, - 0xc75809c42c684dd1, 0x52c07b78a3e60868, - 0xf92e0c3537826145, 0xa7709a56ccdf8a82, - 0x9bbcc7a142b17ccb, 0x88a66076400bb691, - 0xc2abf989935ddbfe, 0x6acff893d00ea435, - 0xf356f7ebf83552fe, 0x583f6b8c4124d43, - 0x98165af37b2153de, 0xc3727a337a8b704a, - 0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5c, - 0xeda2ee1c7064130c, 0x1162def06f79df73, - 0x9485d4d1c63e8be7, 0x8addcb5645ac2ba8, - 0xb9a74a0637ce2ee1, 0x6d953e2bd7173692, - 0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0437, - 0x910ab1d4db9914a0, 0x1d9c9892400a22a2, - 0xb54d5e4a127f59c8, 0x2503beb6d00cab4b, - 0xe2a0b5dc971f303a, 0x2e44ae64840fd61d, - 0x8da471a9de737e24, 0x5ceaecfed289e5d2, - 0xb10d8e1456105dad, 0x7425a83e872c5f47, - 0xdd50f1996b947518, 0xd12f124e28f77719, - 0x8a5296ffe33cc92f, 0x82bd6b70d99aaa6f, - 0xace73cbfdc0bfb7b, 0x636cc64d1001550b, - 0xd8210befd30efa5a, 0x3c47f7e05401aa4e, - 0x8714a775e3e95c78, 0x65acfaec34810a71, - 0xa8d9d1535ce3b396, 0x7f1839a741a14d0d, - 0xd31045a8341ca07c, 0x1ede48111209a050, - 0x83ea2b892091e44d, 0x934aed0aab460432, - 0xa4e4b66b68b65d60, 0xf81da84d5617853f, - 0xce1de40642e3f4b9, 0x36251260ab9d668e, - 0x80d2ae83e9ce78f3, 0xc1d72b7c6b426019, - 0xa1075a24e4421730, 0xb24cf65b8612f81f, - 0xc94930ae1d529cfc, 0xdee033f26797b627, - 0xfb9b7cd9a4a7443c, 0x169840ef017da3b1, - 0x9d412e0806e88aa5, 0x8e1f289560ee864e, - 0xc491798a08a2ad4e, 0xf1a6f2bab92a27e2, - 0xf5b5d7ec8acb58a2, 0xae10af696774b1db, - 0x9991a6f3d6bf1765, 0xacca6da1e0a8ef29, - 0xbff610b0cc6edd3f, 0x17fd090a58d32af3, - 0xeff394dcff8a948e, 0xddfc4b4cef07f5b0, - 0x95f83d0a1fb69cd9, 0x4abdaf101564f98e, - 0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f1, - 0xea53df5fd18d5513, 0x84c86189216dc5ed, - 0x92746b9be2f8552c, 0x32fd3cf5b4e49bb4, - 0xb7118682dbb66a77, 0x3fbc8c33221dc2a1, - 0xe4d5e82392a40515, 0xfabaf3feaa5334a, - 0x8f05b1163ba6832d, 0x29cb4d87f2a7400e, - 0xb2c71d5bca9023f8, 0x743e20e9ef511012, - 0xdf78e4b2bd342cf6, 0x914da9246b255416, - 0x8bab8eefb6409c1a, 0x1ad089b6c2f7548e, - 0xae9672aba3d0c320, 0xa184ac2473b529b1, - 0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741e, - 0x8865899617fb1871, 0x7e2fa67c7a658892, - 0xaa7eebfb9df9de8d, 0xddbb901b98feeab7, - 0xd51ea6fa85785631, 0x552a74227f3ea565, - 0x8533285c936b35de, 0xd53a88958f87275f, - 0xa67ff273b8460356, 0x8a892abaf368f137, - 0xd01fef10a657842c, 0x2d2b7569b0432d85, - 0x8213f56a67f6b29b, 0x9c3b29620e29fc73, - 0xa298f2c501f45f42, 0x8349f3ba91b47b8f, - 0xcb3f2f7642717713, 0x241c70a936219a73, - 0xfe0efb53d30dd4d7, 0xed238cd383aa0110, - 0x9ec95d1463e8a506, 0xf4363804324a40aa, - 0xc67bb4597ce2ce48, 0xb143c6053edcd0d5, - 0xf81aa16fdc1b81da, 0xdd94b7868e94050a, - 0x9b10a4e5e9913128, 0xca7cf2b4191c8326, - 0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f0, - 0xf24a01a73cf2dccf, 0xbc633b39673c8cec, - 0x976e41088617ca01, 0xd5be0503e085d813, - 0xbd49d14aa79dbc82, 0x4b2d8644d8a74e18, - 0xec9c459d51852ba2, 0xddf8e7d60ed1219e, - 0x93e1ab8252f33b45, 0xcabb90e5c942b503, - 0xb8da1662e7b00a17, 0x3d6a751f3b936243, - 0xe7109bfba19c0c9d, 0xcc512670a783ad4, - 0x906a617d450187e2, 0x27fb2b80668b24c5, - 0xb484f9dc9641e9da, 0xb1f9f660802dedf6, - 0xe1a63853bbd26451, 0x5e7873f8a0396973, - 0x8d07e33455637eb2, 0xdb0b487b6423e1e8, - 0xb049dc016abc5e5f, 0x91ce1a9a3d2cda62, - 0xdc5c5301c56b75f7, 0x7641a140cc7810fb, - 0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9d, - 0xac2820d9623bf429, 0x546345fa9fbdcd44, - 0xd732290fbacaf133, 0xa97c177947ad4095, - 0x867f59a9d4bed6c0, 0x49ed8eabcccc485d, - 0xa81f301449ee8c70, 0x5c68f256bfff5a74, - 0xd226fc195c6a2f8c, 0x73832eec6fff3111, - 0x83585d8fd9c25db7, 0xc831fd53c5ff7eab, - 0xa42e74f3d032f525, 0xba3e7ca8b77f5e55, - 0xcd3a1230c43fb26f, 0x28ce1bd2e55f35eb, - 0x80444b5e7aa7cf85, 0x7980d163cf5b81b3, - 0xa0555e361951c366, 0xd7e105bcc332621f, - 0xc86ab5c39fa63440, 0x8dd9472bf3fefaa7, - 0xfa856334878fc150, 0xb14f98f6f0feb951, - 0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d3, - 0xc3b8358109e84f07, 0xa862f80ec4700c8, - 0xf4a642e14c6262c8, 0xcd27bb612758c0fa, - 0x98e7e9cccfbd7dbd, 0x8038d51cb897789c, - 0xbf21e44003acdd2c, 0xe0470a63e6bd56c3, - 0xeeea5d5004981478, 0x1858ccfce06cac74, - 0x95527a5202df0ccb, 0xf37801e0c43ebc8, - 0xbaa718e68396cffd, 0xd30560258f54e6ba, - 0xe950df20247c83fd, 0x47c6b82ef32a2069, - 0x91d28b7416cdd27e, 0x4cdc331d57fa5441, - 0xb6472e511c81471d, 0xe0133fe4adf8e952, - 0xe3d8f9e563a198e5, 0x58180fddd97723a6, - 0x8e679c2f5e44ff8f, 0x570f09eaa7ea7648, - }; -}; - -#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE - -template -constexpr uint64_t - powers_template::power_of_five_128[number_of_entries]; - -#endif - -using powers = powers_template<>; - -} // namespace simdjson_fast_float - -#endif - -#ifndef SIMDJSON_FASTFLOAT_DECIMAL_TO_BINARY_H -#define SIMDJSON_FASTFLOAT_DECIMAL_TO_BINARY_H - -#include -#include -#include -#include -#include -#include - -namespace simdjson_fast_float { - -// This will compute or rather approximate w * 5**q and return a pair of 64-bit -// words approximating the result, with the "high" part corresponding to the -// most significant bits and the low part corresponding to the least significant -// bits. -// -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 value128 -compute_product_approximation(int64_t q, uint64_t w) { - int const index = 2 * int(q - powers::smallest_power_of_five); - // For small values of q, e.g., q in [0,27], the answer is always exact - // because The line value128 firstproduct = full_multiplication(w, - // power_of_five_128[index]); gives the exact answer. - value128 firstproduct = - full_multiplication(w, powers::power_of_five_128[index]); - static_assert((bit_precision >= 0) && (bit_precision <= 64), - " precision should be in (0,64]"); - constexpr uint64_t precision_mask = - (bit_precision < 64) ? (uint64_t(0xFFFFFFFFFFFFFFFF) >> bit_precision) - : uint64_t(0xFFFFFFFFFFFFFFFF); - if ((firstproduct.high & precision_mask) == - precision_mask) { // could further guard with (lower + w < lower) - // regarding the second product, we only need secondproduct.high, but our - // expectation is that the compiler will optimize this extra work away if - // needed. - value128 secondproduct = - full_multiplication(w, powers::power_of_five_128[index + 1]); - firstproduct.low += secondproduct.high; - if (secondproduct.high > firstproduct.low) { - firstproduct.high++; - } - } - return firstproduct; -} - -namespace detail { -/** - * For q in (0,350), we have that - * f = (((152170 + 65536) * q ) >> 16); - * is equal to - * floor(p) + q - * where - * p = log(5**q)/log(2) = q * log(5)/log(2) - * - * For negative values of q in (-400,0), we have that - * f = (((152170 + 65536) * q ) >> 16); - * is equal to - * -ceil(p) + q - * where - * p = log(5**-q)/log(2) = -q * log(5)/log(2) - */ -constexpr simdjson_fastfloat_really_inline int32_t power(int32_t q) noexcept { - return (((152170 + 65536) * q) >> 16) + 63; -} -} // namespace detail - -// create an adjusted mantissa, biased by the invalid power2 -// for significant digits already multiplied by 10 ** q. -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 adjusted_mantissa -compute_error_scaled(int64_t q, uint64_t w, int lz) noexcept { - int hilz = int(w >> 63) ^ 1; - adjusted_mantissa answer; - answer.mantissa = w << hilz; - int bias = binary::mantissa_explicit_bits() - binary::minimum_exponent(); - answer.power2 = int32_t(detail::power(int32_t(q)) + bias - hilz - lz - 62 + - invalid_am_bias); - return answer; -} - -// w * 10 ** q, without rounding the representation up. -// the power2 in the exponent will be adjusted by invalid_am_bias. -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa -compute_error(int64_t q, uint64_t w) noexcept { - int lz = leading_zeroes(w); - w <<= lz; - value128 product = - compute_product_approximation(q, w); - return compute_error_scaled(q, product.high, lz); -} - -// Computers w * 10 ** q. -// The returned value should be a valid number that simply needs to be -// packed. However, in some very rare cases, the computation will fail. In such -// cases, we return an adjusted_mantissa with a negative power of 2: the caller -// should recompute in such cases. -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa -compute_float(int64_t q, uint64_t w) noexcept { - adjusted_mantissa answer; - if ((w == 0) || (q < binary::smallest_power_of_ten())) { - answer.power2 = 0; - answer.mantissa = 0; - // result should be zero - return answer; - } - if (q > binary::largest_power_of_ten()) { - // we want to get infinity: - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; - return answer; - } - // At this point in time q is in [powers::smallest_power_of_five, - // powers::largest_power_of_five]. - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(w); - w <<= lz; - - // The required precision is binary::mantissa_explicit_bits() + 3 because - // 1. We need the implicit bit - // 2. We need an extra bit for rounding purposes - // 3. We might lose a bit due to the "upperbit" routine (result too small, - // requiring a shift) - - value128 product = - compute_product_approximation(q, w); - // The computed 'product' is always sufficient. - // Mathematical proof: - // Noble Mushtak and Daniel Lemire, Fast Number Parsing Without Fallback (to - // appear) See script/mushtak_lemire.py - - // The "compute_product_approximation" function can be slightly slower than a - // branchless approach: value128 product = compute_product(q, w); but in - // practice, we can win big with the compute_product_approximation if its - // additional branch is easily predicted. Which is best is data specific. - int upperbit = int(product.high >> 63); - int shift = upperbit + 64 - binary::mantissa_explicit_bits() - 3; - - answer.mantissa = product.high >> shift; - - answer.power2 = int32_t(detail::power(int32_t(q)) + upperbit - lz - - binary::minimum_exponent()); - if (answer.power2 <= 0) { // we have a subnormal? - // Here have that answer.power2 <= 0 so -answer.power2 >= 0 - if (-answer.power2 + 1 >= - 64) { // if we have more than 64 bits below the minimum exponent, you - // have a zero for sure. - answer.power2 = 0; - answer.mantissa = 0; - // result should be zero - return answer; - } - // next line is safe because -answer.power2 + 1 < 64 - answer.mantissa >>= -answer.power2 + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0 in the 32-bit and - // and 64-bit case (with no more than 19 digits). - answer.mantissa += (answer.mantissa & 1); // round up - answer.mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - answer.power2 = - (answer.mantissa < (uint64_t(1) << binary::mantissa_explicit_bits())) - ? 0 - : 1; - return answer; - } - - // usually, we round *up*, but if we fall right in between and and we have an - // even basis, we need to round down - // We are only concerned with the cases where 5**q fits in single 64-bit word. - if ((product.low <= 1) && (q >= binary::min_exponent_round_to_even()) && - (q <= binary::max_exponent_round_to_even()) && - ((answer.mantissa & 3) == 1)) { // we may fall between two floats! - // To be in-between two floats we need that in doing - // answer.mantissa = product.high >> (upperbit + 64 - - // binary::mantissa_explicit_bits() - 3); - // ... we dropped out only zeroes. But if this happened, then we can go - // back!!! - if ((answer.mantissa << shift) == product.high) { - answer.mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - answer.mantissa += (answer.mantissa & 1); // round up - answer.mantissa >>= 1; - if (answer.mantissa >= (uint64_t(2) << binary::mantissa_explicit_bits())) { - answer.mantissa = (uint64_t(1) << binary::mantissa_explicit_bits()); - answer.power2++; // undo previous addition - } - - answer.mantissa &= ~(uint64_t(1) << binary::mantissa_explicit_bits()); - if (answer.power2 >= binary::infinite_power()) { // infinity - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; - } - return answer; -} - -} // namespace simdjson_fast_float - -#endif - -#ifndef SIMDJSON_FASTFLOAT_BIGINT_H -#define SIMDJSON_FASTFLOAT_BIGINT_H - -#include -#include -#include -#include - - -namespace simdjson_fast_float { - -// the limb width: we want efficient multiplication of double the bits in -// limb, or for 64-bit limbs, at least 64-bit multiplication where we can -// extract the high and low parts efficiently. this is every 64-bit -// architecture except for sparc, which emulates 128-bit multiplication. -// we might have platforms where `CHAR_BIT` is not 8, so let's avoid -// doing `8 * sizeof(limb)`. -#if defined(SIMDJSON_FASTFLOAT_64BIT) && !defined(__sparc) -#define SIMDJSON_FASTFLOAT_64BIT_LIMB 1 -typedef uint64_t limb; -constexpr size_t limb_bits = 64; -#else -#define SIMDJSON_FASTFLOAT_32BIT_LIMB -typedef uint32_t limb; -constexpr size_t limb_bits = 32; -#endif - -typedef span limb_span; - -// number of bits in a bigint. this needs to be at least the number -// of bits required to store the largest bigint, which is -// `log2(10**(digits + max_exp))`, or `log2(10**(767 + 342))`, or -// ~3600 bits, so we round to 4000. -constexpr size_t bigint_bits = 4000; -constexpr size_t bigint_limbs = bigint_bits / limb_bits; - -// vector-like type that is allocated on the stack. the entire -// buffer is pre-allocated, and only the length changes. -template struct stackvec { - limb data[size]; - // we never need more than 150 limbs - uint16_t length{0}; - - stackvec() = default; - stackvec(stackvec const &) = delete; - stackvec &operator=(stackvec const &) = delete; - stackvec(stackvec &&) = delete; - stackvec &operator=(stackvec &&other) = delete; - - // create stack vector from existing limb span. - SIMDJSON_FASTFLOAT_CONSTEXPR20 stackvec(limb_span s) { - SIMDJSON_FASTFLOAT_ASSERT(try_extend(s)); - } - - SIMDJSON_FASTFLOAT_CONSTEXPR14 limb &operator[](size_t index) noexcept { - SIMDJSON_FASTFLOAT_DEBUG_ASSERT(index < length); - return data[index]; - } - - SIMDJSON_FASTFLOAT_CONSTEXPR14 const limb &operator[](size_t index) const noexcept { - SIMDJSON_FASTFLOAT_DEBUG_ASSERT(index < length); - return data[index]; - } - - // index from the end of the container - SIMDJSON_FASTFLOAT_CONSTEXPR14 const limb &rindex(size_t index) const noexcept { - SIMDJSON_FASTFLOAT_DEBUG_ASSERT(index < length); - size_t rindex = length - index - 1; - return data[rindex]; - } - - // set the length, without bounds checking. - SIMDJSON_FASTFLOAT_CONSTEXPR14 void set_len(size_t len) noexcept { - length = uint16_t(len); - } - - constexpr size_t len() const noexcept { return length; } - - constexpr bool is_empty() const noexcept { return length == 0; } - - constexpr size_t capacity() const noexcept { return size; } - - // append item to vector, without bounds checking - SIMDJSON_FASTFLOAT_CONSTEXPR14 void push_unchecked(limb value) noexcept { - data[length] = value; - length++; - } - - // append item to vector, returning if item was added - SIMDJSON_FASTFLOAT_CONSTEXPR14 bool try_push(limb value) noexcept { - if (len() < capacity()) { - push_unchecked(value); - return true; - } else { - return false; - } - } - - // add items to the vector, from a span, without bounds checking - SIMDJSON_FASTFLOAT_CONSTEXPR20 void extend_unchecked(limb_span s) noexcept { - limb *ptr = data + length; - std::copy_n(s.ptr, s.len(), ptr); - set_len(len() + s.len()); - } - - // try to add items to the vector, returning if items were added - SIMDJSON_FASTFLOAT_CONSTEXPR20 bool try_extend(limb_span s) noexcept { - if (len() + s.len() <= capacity()) { - extend_unchecked(s); - return true; - } else { - return false; - } - } - - // resize the vector, without bounds checking - // if the new size is longer than the vector, assign value to each - // appended item. - SIMDJSON_FASTFLOAT_CONSTEXPR20 - void resize_unchecked(size_t new_len, limb value) noexcept { - if (new_len > len()) { - size_t count = new_len - len(); - limb *first = data + len(); - limb *last = first + count; - ::std::fill(first, last, value); - set_len(new_len); - } else { - set_len(new_len); - } - } - - // try to resize the vector, returning if the vector was resized. - SIMDJSON_FASTFLOAT_CONSTEXPR20 bool try_resize(size_t new_len, limb value) noexcept { - if (new_len > capacity()) { - return false; - } else { - resize_unchecked(new_len, value); - return true; - } - } - - // check if any limbs are non-zero after the given index. - // this needs to be done in reverse order, since the index - // is relative to the most significant limbs. - SIMDJSON_FASTFLOAT_CONSTEXPR14 bool nonzero(size_t index) const noexcept { - while (index < len()) { - if (rindex(index) != 0) { - return true; - } - index++; - } - return false; - } - - // normalize the big integer, so most-significant zero limbs are removed. - SIMDJSON_FASTFLOAT_CONSTEXPR14 void normalize() noexcept { - while (len() > 0 && rindex(0) == 0) { - length--; - } - } -}; - -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 uint64_t -empty_hi64(bool &truncated) noexcept { - truncated = false; - return 0; -} - -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t -uint64_hi64(uint64_t r0, bool &truncated) noexcept { - truncated = false; - int shl = leading_zeroes(r0); - return r0 << shl; -} - -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t -uint64_hi64(uint64_t r0, uint64_t r1, bool &truncated) noexcept { - int shl = leading_zeroes(r0); - if (shl == 0) { - truncated = r1 != 0; - return r0; - } else { - int shr = 64 - shl; - truncated = (r1 << shl) != 0; - return (r0 << shl) | (r1 >> shr); - } -} - -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t -uint32_hi64(uint32_t r0, bool &truncated) noexcept { - return uint64_hi64(r0, truncated); -} - -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t -uint32_hi64(uint32_t r0, uint32_t r1, bool &truncated) noexcept { - uint64_t x0 = r0; - uint64_t x1 = r1; - return uint64_hi64((x0 << 32) | x1, truncated); -} - -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t -uint32_hi64(uint32_t r0, uint32_t r1, uint32_t r2, bool &truncated) noexcept { - uint64_t x0 = r0; - uint64_t x1 = r1; - uint64_t x2 = r2; - return uint64_hi64(x0, (x1 << 32) | x2, truncated); -} - -// add two small integers, checking for overflow. -// we want an efficient operation. for msvc, where -// we don't have built-in intrinsics, this is still -// pretty fast. -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 limb -scalar_add(limb x, limb y, bool &overflow) noexcept { - limb z; -// gcc and clang -#if defined(__has_builtin) -#if __has_builtin(__builtin_add_overflow) - if (!cpp20_and_in_constexpr()) { - overflow = __builtin_add_overflow(x, y, &z); - return z; - } -#endif -#endif - - // generic, this still optimizes correctly on MSVC. - z = x + y; - overflow = z < x; - return z; -} - -// multiply two small integers, getting both the high and low bits. -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 limb -scalar_mul(limb x, limb y, limb &carry) noexcept { -#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB -#if defined(__SIZEOF_INT128__) - // GCC and clang both define it as an extension. - __uint128_t z = __uint128_t(x) * __uint128_t(y) + __uint128_t(carry); - carry = limb(z >> limb_bits); - return limb(z); -#else - // fallback, no native 128-bit integer multiplication with carry. - // on msvc, this optimizes identically, somehow. - value128 z = full_multiplication(x, y); - bool overflow; - z.low = scalar_add(z.low, carry, overflow); - z.high += uint64_t(overflow); // cannot overflow - carry = z.high; - return z.low; -#endif -#else - uint64_t z = uint64_t(x) * uint64_t(y) + uint64_t(carry); - carry = limb(z >> limb_bits); - return limb(z); -#endif -} - -// add scalar value to bigint starting from offset. -// used in grade school multiplication -template -inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool small_add_from(stackvec &vec, limb y, - size_t start) noexcept { - size_t index = start; - limb carry = y; - bool overflow; - while (carry != 0 && index < vec.len()) { - vec[index] = scalar_add(vec[index], carry, overflow); - carry = limb(overflow); - index += 1; - } - if (carry != 0) { - SIMDJSON_FASTFLOAT_TRY(vec.try_push(carry)); - } - return true; -} - -// add scalar value to bigint. -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool -small_add(stackvec &vec, limb y) noexcept { - return small_add_from(vec, y, 0); -} - -// multiply bigint by scalar value. -template -inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool small_mul(stackvec &vec, - limb y) noexcept { - limb carry = 0; - for (size_t index = 0; index < vec.len(); index++) { - vec[index] = scalar_mul(vec[index], y, carry); - } - if (carry != 0) { - SIMDJSON_FASTFLOAT_TRY(vec.try_push(carry)); - } - return true; -} - -// add bigint to bigint starting from index. -// used in grade school multiplication -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 bool large_add_from(stackvec &x, limb_span y, - size_t start) noexcept { - // the effective x buffer is from `xstart..x.len()`, so exit early - // if we can't get that current range. - if (x.len() < start || y.len() > x.len() - start) { - SIMDJSON_FASTFLOAT_TRY(x.try_resize(y.len() + start, 0)); - } - - bool carry = false; - for (size_t index = 0; index < y.len(); index++) { - limb xi = x[index + start]; - limb yi = y[index]; - bool c1 = false; - bool c2 = false; - xi = scalar_add(xi, yi, c1); - if (carry) { - xi = scalar_add(xi, 1, c2); - } - x[index + start] = xi; - carry = c1 | c2; - } - - // handle overflow - if (carry) { - SIMDJSON_FASTFLOAT_TRY(small_add_from(x, 1, y.len() + start)); - } - return true; -} - -// add bigint to bigint. -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool -large_add_from(stackvec &x, limb_span y) noexcept { - return large_add_from(x, y, 0); -} - -// grade-school multiplication algorithm -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 bool long_mul(stackvec &x, limb_span y) noexcept { - limb_span xs = limb_span(x.data, x.len()); - stackvec z(xs); - limb_span zs = limb_span(z.data, z.len()); - - if (y.len() != 0) { - limb y0 = y[0]; - SIMDJSON_FASTFLOAT_TRY(small_mul(x, y0)); - for (size_t index = 1; index < y.len(); index++) { - limb yi = y[index]; - stackvec zi; - if (yi != 0) { - // re-use the same buffer throughout - zi.set_len(0); - SIMDJSON_FASTFLOAT_TRY(zi.try_extend(zs)); - SIMDJSON_FASTFLOAT_TRY(small_mul(zi, yi)); - limb_span zis = limb_span(zi.data, zi.len()); - SIMDJSON_FASTFLOAT_TRY(large_add_from(x, zis, index)); - } - } - } - - x.normalize(); - return true; -} - -// grade-school multiplication algorithm -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 bool large_mul(stackvec &x, limb_span y) noexcept { - if (y.len() == 1) { - SIMDJSON_FASTFLOAT_TRY(small_mul(x, y[0])); - } else { - SIMDJSON_FASTFLOAT_TRY(long_mul(x, y)); - } - return true; -} - -template struct pow5_tables { - static constexpr uint32_t large_step = 135; - static constexpr uint64_t small_power_of_5[] = { - 1UL, - 5UL, - 25UL, - 125UL, - 625UL, - 3125UL, - 15625UL, - 78125UL, - 390625UL, - 1953125UL, - 9765625UL, - 48828125UL, - 244140625UL, - 1220703125UL, - 6103515625UL, - 30517578125UL, - 152587890625UL, - 762939453125UL, - 3814697265625UL, - 19073486328125UL, - 95367431640625UL, - 476837158203125UL, - 2384185791015625UL, - 11920928955078125UL, - 59604644775390625UL, - 298023223876953125UL, - 1490116119384765625UL, - 7450580596923828125UL, - }; -#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB - constexpr static limb large_power_of_5[] = { - 1414648277510068013UL, 9180637584431281687UL, 4539964771860779200UL, - 10482974169319127550UL, 198276706040285095UL}; -#else - constexpr static limb large_power_of_5[] = { - 4279965485U, 329373468U, 4020270615U, 2137533757U, 4287402176U, - 1057042919U, 1071430142U, 2440757623U, 381945767U, 46164893U}; -#endif -}; - -#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE - -template constexpr uint32_t pow5_tables::large_step; - -template constexpr uint64_t pow5_tables::small_power_of_5[]; - -template constexpr limb pow5_tables::large_power_of_5[]; - -#endif - -// big integer type. implements a small subset of big integer -// arithmetic, using simple algorithms since asymptotically -// faster algorithms are slower for a small number of limbs. -// all operations assume the big-integer is normalized. -struct bigint : pow5_tables<> { - // storage of the limbs, in little-endian order. - stackvec vec; - - SIMDJSON_FASTFLOAT_CONSTEXPR20 bigint() : vec() {} - - bigint(bigint const &) = delete; - bigint &operator=(bigint const &) = delete; - bigint(bigint &&) = delete; - bigint &operator=(bigint &&other) = delete; - - SIMDJSON_FASTFLOAT_CONSTEXPR20 bigint(uint64_t value) : vec() { -#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB - vec.push_unchecked(value); -#else - vec.push_unchecked(uint32_t(value)); - vec.push_unchecked(uint32_t(value >> 32)); -#endif - vec.normalize(); - } - - // get the high 64 bits from the vector, and if bits were truncated. - // this is to get the significant digits for the float. - SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t hi64(bool &truncated) const noexcept { -#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB - if (vec.len() == 0) { - return empty_hi64(truncated); - } else if (vec.len() == 1) { - return uint64_hi64(vec.rindex(0), truncated); - } else { - uint64_t result = uint64_hi64(vec.rindex(0), vec.rindex(1), truncated); - truncated |= vec.nonzero(2); - return result; - } -#else - if (vec.len() == 0) { - return empty_hi64(truncated); - } else if (vec.len() == 1) { - return uint32_hi64(vec.rindex(0), truncated); - } else if (vec.len() == 2) { - return uint32_hi64(vec.rindex(0), vec.rindex(1), truncated); - } else { - uint64_t result = - uint32_hi64(vec.rindex(0), vec.rindex(1), vec.rindex(2), truncated); - truncated |= vec.nonzero(3); - return result; - } -#endif - } - - // compare two big integers, returning the large value. - // assumes both are normalized. if the return value is - // negative, other is larger, if the return value is - // positive, this is larger, otherwise they are equal. - // the limbs are stored in little-endian order, so we - // must compare the limbs in ever order. - SIMDJSON_FASTFLOAT_CONSTEXPR20 int compare(bigint const &other) const noexcept { - if (vec.len() > other.vec.len()) { - return 1; - } else if (vec.len() < other.vec.len()) { - return -1; - } else { - for (size_t index = vec.len(); index > 0; index--) { - limb xi = vec[index - 1]; - limb yi = other.vec[index - 1]; - if (xi > yi) { - return 1; - } else if (xi < yi) { - return -1; - } - } - return 0; - } - } - - // shift left each limb n bits, carrying over to the new limb - // returns true if we were able to shift all the digits. - SIMDJSON_FASTFLOAT_CONSTEXPR20 bool shl_bits(size_t n) noexcept { - // Internally, for each item, we shift left by n, and add the previous - // right shifted limb-bits. - // For example, we transform (for u8) shifted left 2, to: - // b10100100 b01000010 - // b10 b10010001 b00001000 - SIMDJSON_FASTFLOAT_DEBUG_ASSERT(n != 0); - SIMDJSON_FASTFLOAT_DEBUG_ASSERT(n < sizeof(limb) * 8); - - size_t shl = n; - size_t shr = limb_bits - shl; - limb prev = 0; - for (size_t index = 0; index < vec.len(); index++) { - limb xi = vec[index]; - vec[index] = (xi << shl) | (prev >> shr); - prev = xi; - } - - limb carry = prev >> shr; - if (carry != 0) { - return vec.try_push(carry); - } - return true; - } - - // move the limbs left by `n` limbs. - SIMDJSON_FASTFLOAT_CONSTEXPR20 bool shl_limbs(size_t n) noexcept { - SIMDJSON_FASTFLOAT_DEBUG_ASSERT(n != 0); - if (n + vec.len() > vec.capacity()) { - return false; - } else if (!vec.is_empty()) { - // move limbs - limb *dst = vec.data + n; - limb const *src = vec.data; - std::copy_backward(src, src + vec.len(), dst + vec.len()); - // fill in empty limbs - limb *first = vec.data; - limb *last = first + n; - ::std::fill(first, last, 0); - vec.set_len(n + vec.len()); - return true; - } else { - return true; - } - } - - // move the limbs left by `n` bits. - SIMDJSON_FASTFLOAT_CONSTEXPR20 bool shl(size_t n) noexcept { - size_t rem = n % limb_bits; - size_t div = n / limb_bits; - if (rem != 0) { - SIMDJSON_FASTFLOAT_TRY(shl_bits(rem)); - } - if (div != 0) { - SIMDJSON_FASTFLOAT_TRY(shl_limbs(div)); - } - return true; - } - - // get the number of leading zeros in the bigint. - SIMDJSON_FASTFLOAT_CONSTEXPR20 int ctlz() const noexcept { - if (vec.is_empty()) { - return 0; - } else { -#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB - return leading_zeroes(vec.rindex(0)); -#else - // no use defining a specialized leading_zeroes for a 32-bit type. - uint64_t r0 = vec.rindex(0); - return leading_zeroes(r0 << 32); -#endif - } - } - - // get the number of bits in the bigint. - SIMDJSON_FASTFLOAT_CONSTEXPR20 int bit_length() const noexcept { - int lz = ctlz(); - return int(limb_bits * vec.len()) - lz; - } - - SIMDJSON_FASTFLOAT_CONSTEXPR20 bool mul(limb y) noexcept { return small_mul(vec, y); } - - SIMDJSON_FASTFLOAT_CONSTEXPR20 bool add(limb y) noexcept { return small_add(vec, y); } - - // multiply as if by 2 raised to a power. - SIMDJSON_FASTFLOAT_CONSTEXPR20 bool pow2(uint32_t exp) noexcept { return shl(exp); } - - // multiply as if by 5 raised to a power. - SIMDJSON_FASTFLOAT_CONSTEXPR20 bool pow5(uint32_t exp) noexcept { - // multiply by a power of 5 - size_t large_length = sizeof(large_power_of_5) / sizeof(limb); - limb_span large = limb_span(large_power_of_5, large_length); - while (exp >= large_step) { - SIMDJSON_FASTFLOAT_TRY(large_mul(vec, large)); - exp -= large_step; - } -#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB - uint32_t small_step = 27; - limb max_native = 7450580596923828125UL; -#else - uint32_t small_step = 13; - limb max_native = 1220703125U; -#endif - while (exp >= small_step) { - SIMDJSON_FASTFLOAT_TRY(small_mul(vec, max_native)); - exp -= small_step; - } - if (exp != 0) { - // Work around clang bug https://godbolt.org/z/zedh7rrhc - // This is similar to https://github.com/llvm/llvm-project/issues/47746, - // except the workaround described there don't work here - SIMDJSON_FASTFLOAT_TRY(small_mul(vec, limb((static_cast(small_power_of_5[0]), - small_power_of_5[exp])))); - } - - return true; - } - - // multiply as if by 10 raised to a power. - SIMDJSON_FASTFLOAT_CONSTEXPR20 bool pow10(uint32_t exp) noexcept { - SIMDJSON_FASTFLOAT_TRY(pow5(exp)); - return pow2(exp); - } -}; - -} // namespace simdjson_fast_float - -#endif - -#ifndef SIMDJSON_FASTFLOAT_DIGIT_COMPARISON_H -#define SIMDJSON_FASTFLOAT_DIGIT_COMPARISON_H - -#include -#include -#include - - -namespace simdjson_fast_float { - -// 1e0 to 1e19 -constexpr static uint64_t powers_of_ten_uint64[] = {1UL, - 10UL, - 100UL, - 1000UL, - 10000UL, - 100000UL, - 1000000UL, - 10000000UL, - 100000000UL, - 1000000000UL, - 10000000000UL, - 100000000000UL, - 1000000000000UL, - 10000000000000UL, - 100000000000000UL, - 1000000000000000UL, - 10000000000000000UL, - 100000000000000000UL, - 1000000000000000000UL, - 10000000000000000000UL}; - -// calculate the exponent, in scientific notation, of the number. -// this algorithm is not even close to optimized, but it has no practical -// effect on performance: in order to have a faster algorithm, we'd need -// to slow down performance for faster algorithms, and this is still fast. -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 int32_t -scientific_exponent(uint64_t mantissa, int32_t exponent) noexcept { - while (mantissa >= 10000) { - mantissa /= 10000; - exponent += 4; - } - while (mantissa >= 100) { - mantissa /= 100; - exponent += 2; - } - while (mantissa >= 10) { - mantissa /= 10; - exponent += 1; - } - return exponent; -} - -// this converts a native floating-point number to an extended-precision float. -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa -to_extended(T value) noexcept { - using equiv_uint = equiv_uint_t; - constexpr equiv_uint exponent_mask = binary_format::exponent_mask(); - constexpr equiv_uint mantissa_mask = binary_format::mantissa_mask(); - constexpr equiv_uint hidden_bit_mask = binary_format::hidden_bit_mask(); - - adjusted_mantissa am; - int32_t bias = binary_format::mantissa_explicit_bits() - - binary_format::minimum_exponent(); - equiv_uint bits; -#if SIMDJSON_FASTFLOAT_HAS_BIT_CAST - bits = std::bit_cast(value); -#else - ::memcpy(&bits, &value, sizeof(T)); -#endif - if ((bits & exponent_mask) == 0) { - // denormal - am.power2 = 1 - bias; - am.mantissa = bits & mantissa_mask; - } else { - // normal - am.power2 = int32_t((bits & exponent_mask) >> - binary_format::mantissa_explicit_bits()); - am.power2 -= bias; - am.mantissa = (bits & mantissa_mask) | hidden_bit_mask; - } - - return am; -} - -// get the extended precision value of the halfway point between b and b+u. -// we are given a native float that represents b, so we need to adjust it -// halfway between b and b+u. -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa -to_extended_halfway(T value) noexcept { - adjusted_mantissa am = to_extended(value); - am.mantissa <<= 1; - am.mantissa += 1; - am.power2 -= 1; - return am; -} - -// round an extended-precision float to the nearest machine float. -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 void round(adjusted_mantissa &am, - callback cb) noexcept { - int32_t mantissa_shift = 64 - binary_format::mantissa_explicit_bits() - 1; - if (-am.power2 >= mantissa_shift) { - // have a denormal float - int32_t shift = -am.power2 + 1; - cb(am, (shift < 64 ? shift : 64)); - // check for round-up: if rounding-nearest carried us to the hidden bit. - am.power2 = (am.mantissa < - (uint64_t(1) << binary_format::mantissa_explicit_bits())) - ? 0 - : 1; - return; - } - - // have a normal float, use the default shift. - cb(am, mantissa_shift); - - // check for carry - if (am.mantissa >= - (uint64_t(2) << binary_format::mantissa_explicit_bits())) { - am.mantissa = (uint64_t(1) << binary_format::mantissa_explicit_bits()); - am.power2++; - } - - // check for infinite: we could have carried to an infinite power - am.mantissa &= ~(uint64_t(1) << binary_format::mantissa_explicit_bits()); - if (am.power2 >= binary_format::infinite_power()) { - am.power2 = binary_format::infinite_power(); - am.mantissa = 0; - } -} - -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 void -round_nearest_tie_even(adjusted_mantissa &am, int32_t shift, - callback cb) noexcept { - uint64_t const mask = (shift == 64) ? UINT64_MAX : (uint64_t(1) << shift) - 1; - uint64_t const halfway = (shift == 0) ? 0 : uint64_t(1) << (shift - 1); - uint64_t truncated_bits = am.mantissa & mask; - bool is_above = truncated_bits > halfway; - bool is_halfway = truncated_bits == halfway; - - // shift digits into position - if (shift == 64) { - am.mantissa = 0; - } else { - am.mantissa >>= shift; - } - am.power2 += shift; - - bool is_odd = (am.mantissa & 1) == 1; - am.mantissa += uint64_t(cb(is_odd, is_halfway, is_above)); -} - -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 void -round_down(adjusted_mantissa &am, int32_t shift) noexcept { - if (shift == 64) { - am.mantissa = 0; - } else { - am.mantissa >>= shift; - } - am.power2 += shift; -} - -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void -skip_zeros(UC const *&first, UC const *last) noexcept { - uint64_t val; - while (!cpp20_and_in_constexpr() && - std::distance(first, last) >= int_cmp_len()) { - ::memcpy(&val, first, sizeof(uint64_t)); - if (val != int_cmp_zeros()) { - break; - } - first += int_cmp_len(); - } - while (first != last) { - if (*first != UC('0')) { - break; - } - first++; - } -} - -// determine if any non-zero digits were truncated. -// all characters must be valid digits. -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool -is_truncated(UC const *first, UC const *last) noexcept { - // do 8-bit optimizations, can just compare to 8 literal 0s. - uint64_t val; - while (!cpp20_and_in_constexpr() && - std::distance(first, last) >= int_cmp_len()) { - ::memcpy(&val, first, sizeof(uint64_t)); - if (val != int_cmp_zeros()) { - return true; - } - first += int_cmp_len(); - } - while (first != last) { - if (*first != UC('0')) { - return true; - } - ++first; - } - return false; -} - -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool -is_truncated(span s) noexcept { - return is_truncated(s.ptr, s.ptr + s.len()); -} - -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void -parse_eight_digits(UC const *&p, limb &value, size_t &counter, - size_t &count) noexcept { - value = value * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - counter += 8; - count += 8; -} - -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 void -parse_one_digit(UC const *&p, limb &value, size_t &counter, - size_t &count) noexcept { - value = value * 10 + limb(*p - UC('0')); - p++; - counter++; - count++; -} - -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void -add_native(bigint &big, limb power, limb value) noexcept { - big.mul(power); - big.add(value); -} - -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void -round_up_bigint(bigint &big, size_t &count) noexcept { - // need to round-up the digits, but need to avoid rounding - // ....9999 to ...10000, which could cause a false halfway point. - add_native(big, 10, 1); - count++; -} - -// parse the significant digits into a big integer -template -inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void -parse_mantissa(bigint &result, parsed_number_string_t &num, - size_t max_digits, size_t &digits) noexcept { - // try to minimize the number of big integer and scalar multiplication. - // therefore, try to parse 8 digits at a time, and multiply by the largest - // scalar value (9 or 19 digits) for each step. - size_t counter = 0; - digits = 0; - limb value = 0; -#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB - size_t step = 19; -#else - size_t step = 9; -#endif - - // process all integer digits. - UC const *p = num.integer.ptr; - UC const *pend = p + num.integer.len(); - skip_zeros(p, pend); - // process all digits, in increments of step per loop - while (p != pend) { - while ((std::distance(p, pend) >= 8) && (step - counter >= 8) && - (max_digits - digits >= 8)) { - parse_eight_digits(p, value, counter, digits); - } - while (counter < step && p != pend && digits < max_digits) { - parse_one_digit(p, value, counter, digits); - } - if (digits == max_digits) { - // add the temporary value, then check if we've truncated any digits - add_native(result, limb(powers_of_ten_uint64[counter]), value); - bool truncated = is_truncated(p, pend); - if (num.fraction.ptr != nullptr) { - truncated |= is_truncated(num.fraction); - } - if (truncated) { - round_up_bigint(result, digits); - } - return; - } else { - add_native(result, limb(powers_of_ten_uint64[counter]), value); - counter = 0; - value = 0; - } - } - - // add our fraction digits, if they're available. - if (num.fraction.ptr != nullptr) { - p = num.fraction.ptr; - pend = p + num.fraction.len(); - if (digits == 0) { - skip_zeros(p, pend); - } - // process all digits, in increments of step per loop - while (p != pend) { - while ((std::distance(p, pend) >= 8) && (step - counter >= 8) && - (max_digits - digits >= 8)) { - parse_eight_digits(p, value, counter, digits); - } - while (counter < step && p != pend && digits < max_digits) { - parse_one_digit(p, value, counter, digits); - } - if (digits == max_digits) { - // add the temporary value, then check if we've truncated any digits - add_native(result, limb(powers_of_ten_uint64[counter]), value); - bool truncated = is_truncated(p, pend); - if (truncated) { - round_up_bigint(result, digits); - } - return; - } else { - add_native(result, limb(powers_of_ten_uint64[counter]), value); - counter = 0; - value = 0; - } - } - } - - if (counter != 0) { - add_native(result, limb(powers_of_ten_uint64[counter]), value); - } -} - -template -inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa -positive_digit_comp(bigint &bigmant, int32_t exponent) noexcept { - SIMDJSON_FASTFLOAT_ASSERT(bigmant.pow10(uint32_t(exponent))); - adjusted_mantissa answer; - bool truncated; - answer.mantissa = bigmant.hi64(truncated); - int bias = binary_format::mantissa_explicit_bits() - - binary_format::minimum_exponent(); - answer.power2 = bigmant.bit_length() - 64 + bias; - - round(answer, [truncated](adjusted_mantissa &a, int32_t shift) { - round_nearest_tie_even( - a, shift, - [truncated](bool is_odd, bool is_halfway, bool is_above) -> bool { - return is_above || (is_halfway && truncated) || - (is_odd && is_halfway); - }); - }); - - return answer; -} - -// the scaling here is quite simple: we have, for the real digits `m * 10^e`, -// and for the theoretical digits `n * 2^f`. Since `e` is always negative, -// to scale them identically, we do `n * 2^f * 5^-f`, so we now have `m * 2^e`. -// we then need to scale by `2^(f- e)`, and then the two significant digits -// are of the same magnitude. -template -inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa negative_digit_comp( - bigint &bigmant, adjusted_mantissa am, int32_t exponent) noexcept { - bigint &real_digits = bigmant; - int32_t real_exp = exponent; - - // get the value of `b`, rounded down, and get a bigint representation of b+h - adjusted_mantissa am_b = am; - // gcc7 buf: use a lambda to remove the noexcept qualifier bug with - // -Wnoexcept-type. - round(am_b, - [](adjusted_mantissa &a, int32_t shift) { round_down(a, shift); }); - T b; - to_float(false, am_b, b); - adjusted_mantissa theor = to_extended_halfway(b); - bigint theor_digits(theor.mantissa); - int32_t theor_exp = theor.power2; - - // scale real digits and theor digits to be same power. - int32_t pow2_exp = theor_exp - real_exp; - uint32_t pow5_exp = uint32_t(-real_exp); - if (pow5_exp != 0) { - SIMDJSON_FASTFLOAT_ASSERT(theor_digits.pow5(pow5_exp)); - } - if (pow2_exp > 0) { - SIMDJSON_FASTFLOAT_ASSERT(theor_digits.pow2(uint32_t(pow2_exp))); - } else if (pow2_exp < 0) { - SIMDJSON_FASTFLOAT_ASSERT(real_digits.pow2(uint32_t(-pow2_exp))); - } - - // compare digits, and use it to direct rounding - int ord = real_digits.compare(theor_digits); - adjusted_mantissa answer = am; - round(answer, [ord](adjusted_mantissa &a, int32_t shift) { - round_nearest_tie_even( - a, shift, [ord](bool is_odd, bool _, bool __) -> bool { - static_cast(_); // not needed, since we've done our comparison - static_cast(__); // not needed, since we've done our comparison - if (ord > 0) { - return true; - } else if (ord < 0) { - return false; - } else { - return is_odd; - } - }); - }); - - return answer; -} - -// parse the significant digits as a big integer to unambiguously round -// the significant digits. here, we are trying to determine how to round -// an extended float representation close to `b+h`, halfway between `b` -// (the float rounded-down) and `b+u`, the next positive float. this -// algorithm is always correct, and uses one of two approaches. when -// the exponent is positive relative to the significant digits (such as -// 1234), we create a big-integer representation, get the high 64-bits, -// determine if any lower bits are truncated, and use that to direct -// rounding. in case of a negative exponent relative to the significant -// digits (such as 1.2345), we create a theoretical representation of -// `b` as a big-integer type, scaled to the same binary exponent as -// the actual digits. we then compare the big integer representations -// of both, and use that to direct rounding. -template -inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa -digit_comp(parsed_number_string_t &num, adjusted_mantissa am) noexcept { - // remove the invalid exponent bias - am.power2 -= invalid_am_bias; - - int32_t sci_exp = - scientific_exponent(num.mantissa, static_cast(num.exponent)); - size_t max_digits = binary_format::max_digits(); - size_t digits = 0; - bigint bigmant; - parse_mantissa(bigmant, num, max_digits, digits); - // can't underflow, since digits is at most max_digits. - int32_t exponent = sci_exp + 1 - int32_t(digits); - if (exponent >= 0) { - return positive_digit_comp(bigmant, exponent); - } else { - return negative_digit_comp(bigmant, am, exponent); - } -} - -} // namespace simdjson_fast_float - -#endif - -#ifndef SIMDJSON_FASTFLOAT_PARSE_NUMBER_H -#define SIMDJSON_FASTFLOAT_PARSE_NUMBER_H - - -#include -#include -#include -#include - -namespace simdjson_fast_float { - -namespace detail { -/** - * Special case +inf, -inf, nan, infinity, -infinity. - * The case comparisons could be made much faster given that we know that the - * strings a null-free and fixed. - **/ -template -from_chars_result_t - SIMDJSON_FASTFLOAT_CONSTEXPR14 parse_infnan(UC const *first, UC const *last, - T &value, chars_format fmt) noexcept { - from_chars_result_t answer{}; - answer.ptr = first; - answer.ec = std::errc(); // be optimistic - // assume first < last, so dereference without checks; - bool const minusSign = (*first == UC('-')); - // C++17 20.19.3.(7.1) explicitly forbids '+' sign here - if ((*first == UC('-')) || - (uint64_t(fmt & chars_format::allow_leading_plus) && - (*first == UC('+')))) { - ++first; - } - if (last - first >= 3) { - if (simdjson_fastfloat_strncasecmp3(first, str_const_nan())) { - answer.ptr = (first += 3); - value = minusSign ? -std::numeric_limits::quiet_NaN() - : std::numeric_limits::quiet_NaN(); - // Check for possible nan(n-char-seq-opt), C++17 20.19.3.7, - // C11 7.20.1.3.3. At least MSVC produces nan(ind) and nan(snan). - if (first != last && *first == UC('(')) { - for (UC const *ptr = first + 1; ptr != last; ++ptr) { - if (*ptr == UC(')')) { - answer.ptr = ptr + 1; // valid nan(n-char-seq-opt) - break; - } else if (!((UC('a') <= *ptr && *ptr <= UC('z')) || - (UC('A') <= *ptr && *ptr <= UC('Z')) || - (UC('0') <= *ptr && *ptr <= UC('9')) || *ptr == UC('_'))) - break; // forbidden char, not nan(n-char-seq-opt) - } - } - return answer; - } - if (simdjson_fastfloat_strncasecmp3(first, str_const_inf())) { - if ((last - first >= 8) && - simdjson_fastfloat_strncasecmp5(first + 3, str_const_inf() + 3)) { - answer.ptr = first + 8; - } else { - answer.ptr = first + 3; - } - value = minusSign ? -std::numeric_limits::infinity() - : std::numeric_limits::infinity(); - return answer; - } - } - answer.ec = std::errc::invalid_argument; - return answer; -} - -/** - * Returns true if the floating-pointing rounding mode is to 'nearest'. - * It is the default on most system. This function is meant to be inexpensive. - * Credit : @mwalcott3 - */ -simdjson_fastfloat_really_inline bool rounds_to_nearest() noexcept { - // https://lemire.me/blog/2020/06/26/gcc-not-nearest/ -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - return false; -#endif - // See - // A fast function to check your floating-point rounding mode - // https://lemire.me/blog/2022/11/16/a-fast-function-to-check-your-floating-point-rounding-mode/ - // - // This function is meant to be equivalent to : - // prior: #include - // return fegetround() == FE_TONEAREST; - // However, it is expected to be much faster than the fegetround() - // function call. - // - // The volatile keyword prevents the compiler from computing the function - // at compile-time. - // There might be other ways to prevent compile-time optimizations (e.g., - // asm). The value does not need to be std::numeric_limits::min(), any - // small value so that 1 + x should round to 1 would do (after accounting for - // excess precision, as in 387 instructions). - static float volatile fmin = (std::numeric_limits::min)(); - float fmini = fmin; // we copy it so that it gets loaded at most once. -// -// Explanation: -// Only when fegetround() == FE_TONEAREST do we have that -// fmin + 1.0f == 1.0f - fmin. -// -// FE_UPWARD: -// fmin + 1.0f > 1 -// 1.0f - fmin == 1 -// -// FE_DOWNWARD or FE_TOWARDZERO: -// fmin + 1.0f == 1 -// 1.0f - fmin < 1 -// -// Note: This may fail to be accurate if fast-math has been -// enabled, as rounding conventions may not apply. -#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO -#pragma warning(push) -// todo: is there a VS warning? -// see -// https://stackoverflow.com/questions/46079446/is-there-a-warning-for-floating-point-equality-checking-in-visual-studio-2013 -#elif defined(__clang__) -#pragma clang diagnostic push -SIMDJSON_DISABLE_GCC_WARNING(-Wfloat-equal) -#elif defined(__GNUC__) -#pragma GCC diagnostic push -SIMDJSON_DISABLE_GCC_WARNING(-Wfloat-equal) -#endif - return (fmini + 1.0f == 1.0f - fmini); -#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO -#pragma warning(pop) -#elif defined(__clang__) -#pragma clang diagnostic pop -#elif defined(__GNUC__) -#pragma GCC diagnostic pop -#endif -} - -} // namespace detail - -template struct from_chars_caller { - template - SIMDJSON_FASTFLOAT_CONSTEXPR20 static from_chars_result_t - call(UC const *first, UC const *last, T &value, - parse_options_t options) noexcept { - return from_chars_advanced(first, last, value, options); - } -}; - -#ifdef __STDCPP_FLOAT32_T__ -template <> struct from_chars_caller { - template - SIMDJSON_FASTFLOAT_CONSTEXPR20 static from_chars_result_t - call(UC const *first, UC const *last, std::float32_t &value, - parse_options_t options) noexcept { - // if std::float32_t is defined, and we are in C++23 mode; macro set for - // float32; set value to float due to equivalence between float and - // float32_t - float val = 0.0f; - auto ret = from_chars_advanced(first, last, val, options); - value = val; - return ret; - } -}; -#endif - -#ifdef __STDCPP_FLOAT64_T__ -template <> struct from_chars_caller { - template - SIMDJSON_FASTFLOAT_CONSTEXPR20 static from_chars_result_t - call(UC const *first, UC const *last, std::float64_t &value, - parse_options_t options) noexcept { - // if std::float64_t is defined, and we are in C++23 mode; macro set for - // float64; set value as double due to equivalence between double and - // float64_t - double val = 0.0; - auto ret = from_chars_advanced(first, last, val, options); - value = val; - return ret; - } -}; -#endif - -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars(UC const *first, UC const *last, T &value, - chars_format fmt /*= chars_format::general*/) noexcept { - return from_chars_caller::call(first, last, value, - parse_options_t(fmt)); -} - -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool -clinger_fast_path_impl(uint64_t mantissa, int64_t exponent, bool is_negative, - T &value) noexcept { - // The implementation of the Clinger's fast path is convoluted because - // we want round-to-nearest in all cases, irrespective of the rounding mode - // selected on the thread. - // We proceed optimistically, assuming that detail::rounds_to_nearest() - // returns true. - if (binary_format::min_exponent_fast_path() <= exponent && - exponent <= binary_format::max_exponent_fast_path() && - mantissa <= binary_format::max_mantissa_fast_path()) { - // The mantissa bound above is a necessary condition for BOTH branches - // below: the rounding-mode-dependent branch checks the tighter - // max_mantissa_fast_path(exponent) <= max_mantissa_fast_path(). Testing - // it before detail::rounds_to_nearest() spares long-mantissa inputs - // (which can never take the fast path) the volatile-float probe. - // - // Unfortunately, the conventional Clinger's fast path is only possible - // when the system rounds to the nearest float. - // - // We expect the next branch to almost always be selected. - // We could check it first (before the previous branch), but - // there might be performance advantages at having the check - // be last. - if (!cpp20_and_in_constexpr() && detail::rounds_to_nearest()) { - // We have that fegetround() == FE_TONEAREST. - // Next is Clinger's fast path. - value = T(mantissa); - if (exponent < 0) { - value = value / binary_format::exact_power_of_ten(-exponent); - } else { - value = value * binary_format::exact_power_of_ten(exponent); - } - if (is_negative) { - value = -value; - } - return true; - } else { - // We do not have that fegetround() == FE_TONEAREST. - // Next is a modified Clinger's fast path, inspired by Jakub Jelinek's - // proposal - if (exponent >= 0 && - mantissa <= binary_format::max_mantissa_fast_path(exponent)) { -#if defined(__clang__) || defined(SIMDJSON_FASTFLOAT_32BIT) - // Clang may map 0 to -0.0 when fegetround() == FE_DOWNWARD - if (mantissa == 0) { - value = is_negative ? T(-0.) : T(0.); - return true; - } -#endif - value = T(mantissa) * binary_format::exact_power_of_ten(exponent); - if (is_negative) { - value = -value; - } - return true; - } - } - } - return false; -} - -/** - * This function overload takes parsed_number_string_t structure that is created - * and populated either by from_chars_advanced function taking chars range and - * parsing options or other parsing custom function implemented by user. - */ -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars_advanced(parsed_number_string_t &pns, T &value) noexcept { - static_assert(is_supported_float_type::value, - "only some floating-point types are supported"); - static_assert(is_supported_char_type::value, - "only char, wchar_t, char16_t and char32_t are supported"); - - from_chars_result_t answer; - - answer.ec = std::errc(); // be optimistic - answer.ptr = pns.lastmatch; - - if (!pns.too_many_digits && - clinger_fast_path_impl(pns.mantissa, pns.exponent, pns.negative, value)) - return answer; - - adjusted_mantissa am = - compute_float>(pns.exponent, pns.mantissa); - if (pns.too_many_digits && am.power2 >= 0) { - if (am != compute_float>(pns.exponent, pns.mantissa + 1)) { - am = compute_error>(pns.exponent, pns.mantissa); - } - } - // If we called compute_float>(pns.exponent, pns.mantissa) - // and we have an invalid power (am.power2 < 0), then we need to go the long - // way around again. This is very uncommon. - if (am.power2 < 0) { - am = digit_comp(pns, am); - } - to_float(pns.negative, am, value); - // Test for over/underflow. - if ((pns.mantissa != 0 && am.mantissa == 0 && am.power2 == 0) || - am.power2 == binary_format::infinite_power()) { - answer.ec = std::errc::result_out_of_range; - } - return answer; -} - -// Slow path: re-parse materializing the integer/fraction spans the hot no-span -// parse skipped, then run the full algorithm. The two callers reach it only -// through a simdjson_fastfloat_unlikely branch, so the optimizer keeps this re-parse off -// the hot path on its own (no function-level noinline needed). -// from_chars_advanced already handles both the too_many_digits disambiguation -// and the am.power2<0 digit_comp recompute, so both slow branches collapse to -// one helper call. -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t -parse_number_slow_path(UC const *first, UC const *last, T &value, - parse_options_t options, bool bjf) noexcept { - parsed_number_string_t pns = - bjf ? parse_number_string(first, last, options, true) - : parse_number_string(first, last, options, true); - return from_chars_advanced(pns, value); -} - -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars_float_advanced(UC const *first, UC const *last, T &value, - parse_options_t options) noexcept { - - static_assert(is_supported_float_type::value, - "only some floating-point types are supported"); - static_assert(is_supported_char_type::value, - "only char, wchar_t, char16_t and char32_t are supported"); - - chars_format const fmt = detail::adjust_for_feature_macros(options.format); - - from_chars_result_t answer; - if (uint64_t(fmt & chars_format::skip_white_space)) { - while ((first != last) && simdjson_fast_float::is_space(*first)) { - first++; - } - } - if (first == last) { - answer.ec = std::errc::invalid_argument; - answer.ptr = first; - return answer; - } - bool const bjf = uint64_t(fmt & detail::basic_json_fmt) != 0; - - // Fast path: parse WITHOUT materializing the integer/fraction spans (read - // only by the rare slow paths). Skipping their stores keeps the fat - // parsed_number_string_t off the hot path. store_spans is a runtime argument, - // so this reuses the single parse_number_string instantiation. - parsed_number_string_t pns = - bjf ? parse_number_string(first, last, options, false) - : parse_number_string(first, last, options, false); - if (!pns.valid) { - if (uint64_t(fmt & chars_format::no_infnan)) { - answer.ec = std::errc::invalid_argument; - answer.ptr = first; - return answer; - } else { - return detail::parse_infnan(first, last, value, fmt); - } - } - - // Slow path A (rare): > 19 significant digits. The no-span parse left the - // mantissa un-truncated and skipped the span-based recompute; the cold helper - // re-parses with spans and runs the full algorithm. - // -// We have to disable -Wc++20-extensions for the [[unlikely]] attribute -// See comment for @jwakely at -// https://github.com/fastfloat/simdjson_fast_float/pull/387#discussion_r3366943539 -// This is unfortunate. -#ifdef __clang__ -#pragma clang diagnostic push -#if (!defined(__APPLE_CC__) && __clang_major__ >= 10) || (__clang_major__ >= 13) -SIMDJSON_DISABLE_GCC_WARNING(-Wc++20-extensions) -#endif -#endif - if simdjson_fastfloat_unlikely (pns.too_many_digits) { - return parse_number_slow_path(first, last, value, options, bjf); - } - answer.ec = std::errc(); // be optimistic - answer.ptr = pns.lastmatch; - - if (clinger_fast_path_impl(pns.mantissa, pns.exponent, pns.negative, value)) { - return answer; - } - - adjusted_mantissa am = - compute_float>(pns.exponent, pns.mantissa); - // Slow path B (rare): Eisel-Lemire could not resolve; digit_comp needs the - // integer/fraction spans. Route to the cold helper (clinger there is a - // dead-effect since it already failed here; the cold re-parse + digit_comp - // via from_chars_advanced reproduces this branch). - if simdjson_fastfloat_unlikely (am.power2 < 0) { - return parse_number_slow_path(first, last, value, options, bjf); - } -#ifdef __clang__ -#pragma clang diagnostic pop -#endif - to_float(pns.negative, am, value); - // Test for over/underflow. - if ((pns.mantissa != 0 && am.mantissa == 0 && am.power2 == 0) || - am.power2 == binary_format::infinite_power()) { - answer.ec = std::errc::result_out_of_range; - } - return answer; -} - -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars(UC const *first, UC const *last, T &value, int base) noexcept { - - static_assert(is_supported_integer_type::value, - "only integer types are supported"); - static_assert(is_supported_char_type::value, - "only char, wchar_t, char16_t and char32_t are supported"); - - parse_options_t options; - options.base = base; - return from_chars_advanced(first, last, value, options); -} - -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 - typename std::enable_if::value, T>::type - integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept { - T value; - if (clinger_fast_path_impl(mantissa, decimal_exponent, false, value)) - return value; - - adjusted_mantissa am = - compute_float>(decimal_exponent, mantissa); - to_float(false, am, value); - return value; -} - -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 - typename std::enable_if::value, T>::type - integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept { - const bool is_negative = mantissa < 0; - const uint64_t m = static_cast(is_negative ? -mantissa : mantissa); - - T value; - if (clinger_fast_path_impl(m, decimal_exponent, is_negative, value)) - return value; - - adjusted_mantissa am = compute_float>(decimal_exponent, m); - to_float(is_negative, am, value); - return value; -} - -SIMDJSON_FASTFLOAT_CONSTEXPR20 inline double -integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept { - return integer_times_pow10(mantissa, decimal_exponent); -} - -SIMDJSON_FASTFLOAT_CONSTEXPR20 inline double -integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept { - return integer_times_pow10(mantissa, decimal_exponent); -} - -// the following overloads are here to avoid surprising ambiguity for int, -// unsigned, etc. -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 - typename std::enable_if::value && - std::is_integral::value && - !std::is_signed::value, - T>::type - integer_times_pow10(Int mantissa, int decimal_exponent) noexcept { - return integer_times_pow10(static_cast(mantissa), - decimal_exponent); -} - -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 - typename std::enable_if::value && - std::is_integral::value && - std::is_signed::value, - T>::type - integer_times_pow10(Int mantissa, int decimal_exponent) noexcept { - return integer_times_pow10(static_cast(mantissa), - decimal_exponent); -} - -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 typename std::enable_if< - std::is_integral::value && !std::is_signed::value, double>::type -integer_times_pow10(Int mantissa, int decimal_exponent) noexcept { - return integer_times_pow10(static_cast(mantissa), decimal_exponent); -} - -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 typename std::enable_if< - std::is_integral::value && std::is_signed::value, double>::type -integer_times_pow10(Int mantissa, int decimal_exponent) noexcept { - return integer_times_pow10(static_cast(mantissa), decimal_exponent); -} - -template -SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars_int_advanced(UC const *first, UC const *last, T &value, - parse_options_t options) noexcept { - - static_assert(is_supported_integer_type::value, - "only integer types are supported"); - static_assert(is_supported_char_type::value, - "only char, wchar_t, char16_t and char32_t are supported"); - - chars_format const fmt = detail::adjust_for_feature_macros(options.format); - int const base = options.base; - - from_chars_result_t answer; - if (uint64_t(fmt & chars_format::skip_white_space)) { - while ((first != last) && simdjson_fast_float::is_space(*first)) { - first++; - } - } - if (first == last || base < 2 || base > 36) { - answer.ec = std::errc::invalid_argument; - answer.ptr = first; - return answer; - } - - return parse_int_string(first, last, value, options); -} - -template struct from_chars_advanced_caller { - static_assert(TypeIx > 0, "unsupported type"); -}; - -template <> struct from_chars_advanced_caller<1> { - template - simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 static from_chars_result_t - call(UC const *first, UC const *last, T &value, - parse_options_t options) noexcept { - return from_chars_float_advanced(first, last, value, options); - } -}; - -template <> struct from_chars_advanced_caller<2> { - template - simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 static from_chars_result_t - call(UC const *first, UC const *last, T &value, - parse_options_t options) noexcept { - return from_chars_int_advanced(first, last, value, options); - } -}; - -template -simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t -from_chars_advanced(UC const *first, UC const *last, T &value, - parse_options_t options) noexcept { - return from_chars_advanced_caller< - size_t(is_supported_float_type::value) + - 2 * size_t(is_supported_integer_type::value)>::call(first, last, value, - options); -} - -} // namespace simdjson_fast_float - -#endif - -/* end file simdjson/internal/fast_float.h */ - -#include -#include -#include - -namespace simdjson { -namespace internal { - -/** - * These functions handle floating-point parsing when the fast path in - * numberparsing.h gives up: more than 19 digits in the decimal mantissa, an - * exponent outside the range the power-of-five table covers, or one of the rare - * inputs where the truncated Eisel-Lemire product is not accurate enough to - * round. That should only be seen in adversarial scenarios; we do not expect - * production systems to even produce such floating-point numbers. - * - * The work is handed to fast_float (vendored in - * include/simdjson/internal/fast_float.h), which settles the rounding by - * comparing a bigint against a scaled power of five. It is correctly rounded, - * and quick enough that an adversarial document is no longer worth worrying - * about. - **/ - -namespace { - -// fast_float wants the end of the number, and the callers only promise that a -// number is followed by a character which cannot be part of one -- the input -// has already been validated against the JSON grammar, and the buffer is padded, -// so such a character is always there to be found. Locating it costs a pass over -// digits we are about to parse anyway, and in exchange fast_float can bound its -// inner loops instead of re-checking a far-away end pointer. -const char *find_end_of_number(const char *first) noexcept { - const char *p = first; - while ((*p >= '0' && *p <= '9') || *p == '-' || *p == '+' || *p == '.' || - *p == 'e' || *p == 'E') { - p++; - } - return p; -} - -// The input is JSON, so parse it under the JSON grammar: no hexadecimal, no -// leading plus, and no infinity or NaN spellings. Those are handled (or -// rejected) before we ever get here. -constexpr simdjson_fast_float::parse_options json_options{ - simdjson_fast_float::chars_format::json}; - -// fast_float reports result_out_of_range for a value at either edge of the -// format, writing +/-0 when it underflows and +/-infinity when it overflows. -// Both are exactly what the callers of these functions expect to receive: they -// accept a zero and treat an infinity as an error. A malformed number cannot -// happen on validated input, but if it somehow did, returning zero matches what -// the previous implementation did with digits it could not use. -template T parse_with_fast_float(const char *first, const char *end) noexcept { - T value{}; - auto answer = - simdjson_fast_float::from_chars_advanced(first, end, value, json_options); - if (answer.ec == std::errc::invalid_argument) { return T(0); } - return value; -} - -} // namespace - -double from_chars(const char *first) noexcept { - return parse_with_fast_float(first, find_end_of_number(first)); -} - -double from_chars(const char *first, const char *end) noexcept { - return parse_with_fast_float(first, end); -} - -float from_chars_float(const char *first) noexcept { - return parse_with_fast_float(first, find_end_of_number(first)); -} - -} // internal -} // simdjson - -#endif // SIMDJSON_SRC_FROM_CHARS_CPP -/* end file from_chars.cpp */ -/* including internal/error_tables.cpp: #include */ -/* begin file internal/error_tables.cpp */ -#ifndef SIMDJSON_SRC_ERROR_TABLES_CPP -#define SIMDJSON_SRC_ERROR_TABLES_CPP - -/* including simdjson/internal/jsoncharutils_tables.h: #include */ -/* begin file simdjson/internal/jsoncharutils_tables.h */ -#ifndef SIMDJSON_INTERNAL_JSONCHARUTILS_TABLES_H -#define SIMDJSON_INTERNAL_JSONCHARUTILS_TABLES_H - -/* skipped duplicate #include "simdjson/base.h" */ - -#ifdef JSON_TEST_STRINGS -void found_string(const uint8_t *buf, const uint8_t *parsed_begin, - const uint8_t *parsed_end); -void found_bad_string(const uint8_t *buf); -#endif - -namespace simdjson { -namespace internal { -// structural chars here are -// they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c (and NULL) -// we are also interested in the four whitespace characters -// space 0x20, linefeed 0x0a, horizontal tab 0x09 and carriage return 0x0d - -extern SIMDJSON_DLLIMPORTEXPORT const bool structural_or_whitespace_negated[256]; -extern SIMDJSON_DLLIMPORTEXPORT const bool structural_or_whitespace[256]; -extern SIMDJSON_DLLIMPORTEXPORT const uint32_t digit_to_val32[886]; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_JSONCHARUTILS_TABLES_H -/* end file simdjson/internal/jsoncharutils_tables.h */ -/* including simdjson/error-inl.h: #include */ -/* begin file simdjson/error-inl.h */ -#ifndef SIMDJSON_ERROR_INL_H -#define SIMDJSON_ERROR_INL_H - -/* skipped duplicate #include "simdjson/error.h" */ - -#include - -namespace simdjson { - -inline bool is_fatal(error_code error) noexcept { - return error == TAPE_ERROR || error == INCOMPLETE_ARRAY_OR_OBJECT || error == OUT_OF_ORDER_ITERATION || error == DEPTH_ERROR; -} - -namespace internal { - // We store the error code so we can validate the error message is associated with the right code - struct error_code_info { - error_code code; - const char* message; // do not use a fancy std::string where a simple C string will do (no alloc, no destructor) - }; - // These MUST match the codes in error_code. We check this constraint in basictests. - extern SIMDJSON_DLLIMPORTEXPORT const error_code_info error_codes[]; -} // namespace internal - - -inline const char *error_message(error_code error) noexcept { - // If you're using error_code, we're trusting you got it from the enum. - return internal::error_codes[int(error)].message; -} - -// deprecated function -#ifndef SIMDJSON_DISABLE_DEPRECATED_API -inline const std::string error_message(int error) noexcept { - if (error < 0 || error >= error_code::NUM_ERROR_CODES) { - return internal::error_codes[UNEXPECTED_ERROR].message; - } - return internal::error_codes[error].message; -} -#endif // SIMDJSON_DISABLE_DEPRECATED_API - -inline std::ostream& operator<<(std::ostream& out, error_code error) noexcept { - return out << error_message(error); -} - -namespace internal { - -// -// internal::simdjson_result_base inline implementation -// - -template -simdjson_inline void simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_inline error_code simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_inline bool simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - - -template -simdjson_inline T& simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - - -template -simdjson_inline const T& simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T&& simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline simdjson_result_base::simdjson_result_base(T &&value, error_code error) noexcept - : std::pair(std::forward(value), error) {} -template -simdjson_inline simdjson_result_base::simdjson_result_base(error_code error) noexcept - : simdjson_result_base(T{}, error) {} -template -simdjson_inline simdjson_result_base::simdjson_result_base(T &&value) noexcept - : simdjson_result_base(std::forward(value), SUCCESS) {} -template -simdjson_inline simdjson_result_base::simdjson_result_base() noexcept - : simdjson_result_base(T{}, UNINITIALIZED) {} - -} // namespace internal - -/// -/// simdjson_result inline implementation -/// - -template -simdjson_inline void simdjson_result::tie(T &value, error_code &error) && noexcept { - std::forward>(*this).tie(value, error); -} - -template -simdjson_warn_unused simdjson_inline error_code -simdjson_result::get(T &value) && noexcept { - return std::forward>(*this).get(value); -} - -template -simdjson_inline error_code simdjson_result::error() const noexcept { - return internal::simdjson_result_base::error(); -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& simdjson_result::value() & noexcept(false) { - return internal::simdjson_result_base::value(); -} - -template -simdjson_inline T&& simdjson_result::value() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T&& simdjson_result::take_value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline simdjson_result::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& simdjson_result::value_unsafe() const& noexcept { - return internal::simdjson_result_base::value_unsafe(); -} - -template -simdjson_inline T&& simdjson_result::value_unsafe() && noexcept { - return std::forward>(*this).value_unsafe(); -} - -template -simdjson_inline simdjson_result::simdjson_result(T &&value, error_code error) noexcept - : internal::simdjson_result_base(std::forward(value), error) {} -template -simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept - : internal::simdjson_result_base(error) {} -template -simdjson_inline simdjson_result::simdjson_result(T &&value) noexcept - : internal::simdjson_result_base(std::forward(value)) {} -template -simdjson_inline simdjson_result::simdjson_result() noexcept - : internal::simdjson_result_base() {} - -} // namespace simdjson - -#endif // SIMDJSON_ERROR_INL_H -/* end file simdjson/error-inl.h */ - -namespace simdjson { -namespace internal { - - SIMDJSON_DLLIMPORTEXPORT const error_code_info error_codes[] { - { SUCCESS, "SUCCESS: No error" }, - { CAPACITY, "CAPACITY: This parser can't support a document that big" }, - { MEMALLOC, "MEMALLOC: Error allocating memory, we're most likely out of memory" }, - { TAPE_ERROR, "TAPE_ERROR: The JSON document has an improper structure: missing or superfluous commas, braces, missing keys, etc. This is a fatal and unrecoverable error." }, - { DEPTH_ERROR, "DEPTH_ERROR: The JSON document was too deep (too many nested objects and arrays)" }, - { STRING_ERROR, "STRING_ERROR: Problem while parsing a string" }, - { T_ATOM_ERROR, "T_ATOM_ERROR: Problem while parsing an atom starting with the letter 't'" }, - { F_ATOM_ERROR, "F_ATOM_ERROR: Problem while parsing an atom starting with the letter 'f'" }, - { N_ATOM_ERROR, "N_ATOM_ERROR: Problem while parsing an atom starting with the letter 'n'" }, - { NUMBER_ERROR, "NUMBER_ERROR: Problem while parsing a number" }, - { BIGINT_ERROR, "BIGINT_ERROR: Big integer value that cannot be represented using 64 bits" }, - { UTF8_ERROR, "UTF8_ERROR: The input is not valid UTF-8" }, - { UNINITIALIZED, "UNINITIALIZED: Uninitialized" }, - { EMPTY, "EMPTY: no JSON found" }, - { UNESCAPED_CHARS, "UNESCAPED_CHARS: Within strings, some characters must be escaped, we found unescaped characters" }, - { UNCLOSED_STRING, "UNCLOSED_STRING: A string is opened, but never closed." }, - { UNSUPPORTED_ARCHITECTURE, "UNSUPPORTED_ARCHITECTURE: simdjson does not have an implementation supported by this CPU architecture. Please report this error to the core team as it should never happen." }, - { INCORRECT_TYPE, "INCORRECT_TYPE: The JSON element does not have the requested type." }, - { NUMBER_OUT_OF_RANGE, "NUMBER_OUT_OF_RANGE: The JSON number is too large or too small to fit within the requested type." }, - { INDEX_OUT_OF_BOUNDS, "INDEX_OUT_OF_BOUNDS: Attempted to access an element of a JSON array that is beyond its length." }, - { NO_SUCH_FIELD, "NO_SUCH_FIELD: The JSON field referenced does not exist in this object." }, - { IO_ERROR, "IO_ERROR: Error reading the file." }, - { INVALID_JSON_POINTER, "INVALID_JSON_POINTER: Invalid JSON pointer syntax." }, - { INVALID_URI_FRAGMENT, "INVALID_URI_FRAGMENT: Invalid URI fragment syntax." }, - { UNEXPECTED_ERROR, "UNEXPECTED_ERROR: Unexpected error, consider reporting this problem as you may have found a bug in simdjson" }, - { PARSER_IN_USE, "PARSER_IN_USE: Cannot parse a new document while a document is still in use." }, - { OUT_OF_ORDER_ITERATION, "OUT_OF_ORDER_ITERATION: Objects and arrays can only be iterated when they are first encountered." }, - { INSUFFICIENT_PADDING, "INSUFFICIENT_PADDING: simdjson requires the input JSON string to have at least SIMDJSON_PADDING extra bytes allocated, beyond the string's length. Consider using the simdjson::padded_string class if needed." }, - { INCOMPLETE_ARRAY_OR_OBJECT, "INCOMPLETE_ARRAY_OR_OBJECT: JSON document ended early in the middle of an object or array. This is a fatal and unrecoverable error." }, - { SCALAR_DOCUMENT_AS_VALUE, "SCALAR_DOCUMENT_AS_VALUE: A JSON document made of a scalar (number, Boolean, null or string) is treated as a value. Use get_bool(), get_double(), etc. on the document instead. "}, - { OUT_OF_BOUNDS, "OUT_OF_BOUNDS: Attempt to access location outside of document."}, - { TRAILING_CONTENT, "TRAILING_CONTENT: Unexpected trailing content in the JSON input."}, - { OUT_OF_CAPACITY, "OUT_OF_CAPACITY: The capacity was exceeded, we cannot allocate enough memory."}, - { UNKNOWN_FIELD, "UNKNOWN_FIELD: The JSON object has a field that does not map to any member of the target type (deny_unknown_fields)."} - }; // error_messages[] - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_SRC_ERROR_TABLES_CPP -/* end file internal/error_tables.cpp */ -/* including internal/jsoncharutils_tables.cpp: #include */ -/* begin file internal/jsoncharutils_tables.cpp */ -#ifndef SIMDJSON_SRC_JSONCHARUTILS_TABLES_CPP -#define SIMDJSON_SRC_JSONCHARUTILS_TABLES_CPP - -/* skipped duplicate #include */ - -namespace simdjson { -namespace internal { - -// structural chars here are -// they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c (and NULL) -// we are also interested in the four whitespace characters -// space 0x20, linefeed 0x0a, horizontal tab 0x09 and carriage return 0x0d - -SIMDJSON_DLLIMPORTEXPORT const bool structural_or_whitespace_negated[256] = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; - -SIMDJSON_DLLIMPORTEXPORT const bool structural_or_whitespace[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - -SIMDJSON_DLLIMPORTEXPORT const uint32_t digit_to_val32[886] = { - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, - 0x6, 0x7, 0x8, 0x9, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa, - 0xb, 0xc, 0xd, 0xe, 0xf, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xa, 0xb, 0xc, 0xd, 0xe, - 0xf, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0x0, 0x10, 0x20, 0x30, 0x40, 0x50, - 0x60, 0x70, 0x80, 0x90, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa0, - 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, - 0xf0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0x0, 0x100, 0x200, 0x300, 0x400, 0x500, - 0x600, 0x700, 0x800, 0x900, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa00, - 0xb00, 0xc00, 0xd00, 0xe00, 0xf00, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xa00, 0xb00, 0xc00, 0xd00, 0xe00, - 0xf00, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0x0, 0x1000, 0x2000, 0x3000, 0x4000, 0x5000, - 0x6000, 0x7000, 0x8000, 0x9000, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xa000, - 0xb000, 0xc000, 0xd000, 0xe000, 0xf000, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xa000, 0xb000, 0xc000, 0xd000, 0xe000, - 0xf000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, - 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_SRC_JSONCHARUTILS_TABLES_CPP -/* end file internal/jsoncharutils_tables.cpp */ -/* including internal/numberparsing_tables.cpp: #include */ -/* begin file internal/numberparsing_tables.cpp */ -#ifndef SIMDJSON_SRC_NUMBERPARSING_TABLES_CPP -#define SIMDJSON_SRC_NUMBERPARSING_TABLES_CPP - -/* skipped duplicate #include */ -/* including simdjson/internal/numberparsing_tables.h: #include */ -/* begin file simdjson/internal/numberparsing_tables.h */ -#ifndef SIMDJSON_INTERNAL_NUMBERPARSING_TABLES_H -#define SIMDJSON_INTERNAL_NUMBERPARSING_TABLES_H - -/* skipped duplicate #include "simdjson/base.h" */ - -namespace simdjson { -namespace internal { -/** - * The smallest non-zero float (binary64) is 2^-1074. - * We take as input numbers of the form w x 10^q where w < 2^64. - * We have that w * 10^-343 < 2^(64-344) 5^-343 < 2^-1076. - * However, we have that - * (2^64-1) * 10^-342 = (2^64-1) * 2^-342 * 5^-342 > 2^-1074. - * Thus it is possible for a number of the form w * 10^-342 where - * w is a 64-bit value to be a non-zero floating-point number. - ********* - * Any number of form w * 10^309 where w>= 1 is going to be - * infinite in binary64 so we never need to worry about powers - * of 5 greater than 308. - */ -constexpr int smallest_power = -342; -constexpr int largest_power = 308; - -/** - * Represents a 128-bit value. - * low: least significant 64 bits. - * high: most significant 64 bits. - */ -struct value128 { - uint64_t low; - uint64_t high; -}; - - -// Precomputed powers of ten from 10^0 to 10^22. These -// can be represented exactly using the double type. -extern SIMDJSON_DLLIMPORTEXPORT const double power_of_ten[]; - - -/** - * When mapping numbers from decimal to binary, - * we go from w * 10^q to m * 2^p but we have - * 10^q = 5^q * 2^q, so effectively - * we are trying to match - * w * 2^q * 5^q to m * 2^p. Thus the powers of two - * are not a concern since they can be represented - * exactly using the binary notation, only the powers of five - * affect the binary significand. - */ - - -// The truncated powers of five from 5^-342 all the way to 5^308 -// The mantissa is truncated to 128 bits, and -// never rounded up. Uses about 10KB. -// We use the template trick to allow inclusion in multiple translation units. -#if SIMDJSON_STATIC_REFLECTION -template struct powers_template { -constexpr static uint64_t power_of_five_128[651*2]= { - 0xeef453d6923bd65a,0x113faa2906a13b3f, - 0x9558b4661b6565f8,0x4ac7ca59a424c507, - 0xbaaee17fa23ebf76,0x5d79bcf00d2df649, - 0xe95a99df8ace6f53,0xf4d82c2c107973dc, - 0x91d8a02bb6c10594,0x79071b9b8a4be869, - 0xb64ec836a47146f9,0x9748e2826cdee284, - 0xe3e27a444d8d98b7,0xfd1b1b2308169b25, - 0x8e6d8c6ab0787f72,0xfe30f0f5e50e20f7, - 0xb208ef855c969f4f,0xbdbd2d335e51a935, - 0xde8b2b66b3bc4723,0xad2c788035e61382, - 0x8b16fb203055ac76,0x4c3bcb5021afcc31, - 0xaddcb9e83c6b1793,0xdf4abe242a1bbf3d, - 0xd953e8624b85dd78,0xd71d6dad34a2af0d, - 0x87d4713d6f33aa6b,0x8672648c40e5ad68, - 0xa9c98d8ccb009506,0x680efdaf511f18c2, - 0xd43bf0effdc0ba48,0x212bd1b2566def2, - 0x84a57695fe98746d,0x14bb630f7604b57, - 0xa5ced43b7e3e9188,0x419ea3bd35385e2d, - 0xcf42894a5dce35ea,0x52064cac828675b9, - 0x818995ce7aa0e1b2,0x7343efebd1940993, - 0xa1ebfb4219491a1f,0x1014ebe6c5f90bf8, - 0xca66fa129f9b60a6,0xd41a26e077774ef6, - 0xfd00b897478238d0,0x8920b098955522b4, - 0x9e20735e8cb16382,0x55b46e5f5d5535b0, - 0xc5a890362fddbc62,0xeb2189f734aa831d, - 0xf712b443bbd52b7b,0xa5e9ec7501d523e4, - 0x9a6bb0aa55653b2d,0x47b233c92125366e, - 0xc1069cd4eabe89f8,0x999ec0bb696e840a, - 0xf148440a256e2c76,0xc00670ea43ca250d, - 0x96cd2a865764dbca,0x380406926a5e5728, - 0xbc807527ed3e12bc,0xc605083704f5ecf2, - 0xeba09271e88d976b,0xf7864a44c633682e, - 0x93445b8731587ea3,0x7ab3ee6afbe0211d, - 0xb8157268fdae9e4c,0x5960ea05bad82964, - 0xe61acf033d1a45df,0x6fb92487298e33bd, - 0x8fd0c16206306bab,0xa5d3b6d479f8e056, - 0xb3c4f1ba87bc8696,0x8f48a4899877186c, - 0xe0b62e2929aba83c,0x331acdabfe94de87, - 0x8c71dcd9ba0b4925,0x9ff0c08b7f1d0b14, - 0xaf8e5410288e1b6f,0x7ecf0ae5ee44dd9, - 0xdb71e91432b1a24a,0xc9e82cd9f69d6150, - 0x892731ac9faf056e,0xbe311c083a225cd2, - 0xab70fe17c79ac6ca,0x6dbd630a48aaf406, - 0xd64d3d9db981787d,0x92cbbccdad5b108, - 0x85f0468293f0eb4e,0x25bbf56008c58ea5, - 0xa76c582338ed2621,0xaf2af2b80af6f24e, - 0xd1476e2c07286faa,0x1af5af660db4aee1, - 0x82cca4db847945ca,0x50d98d9fc890ed4d, - 0xa37fce126597973c,0xe50ff107bab528a0, - 0xcc5fc196fefd7d0c,0x1e53ed49a96272c8, - 0xff77b1fcbebcdc4f,0x25e8e89c13bb0f7a, - 0x9faacf3df73609b1,0x77b191618c54e9ac, - 0xc795830d75038c1d,0xd59df5b9ef6a2417, - 0xf97ae3d0d2446f25,0x4b0573286b44ad1d, - 0x9becce62836ac577,0x4ee367f9430aec32, - 0xc2e801fb244576d5,0x229c41f793cda73f, - 0xf3a20279ed56d48a,0x6b43527578c1110f, - 0x9845418c345644d6,0x830a13896b78aaa9, - 0xbe5691ef416bd60c,0x23cc986bc656d553, - 0xedec366b11c6cb8f,0x2cbfbe86b7ec8aa8, - 0x94b3a202eb1c3f39,0x7bf7d71432f3d6a9, - 0xb9e08a83a5e34f07,0xdaf5ccd93fb0cc53, - 0xe858ad248f5c22c9,0xd1b3400f8f9cff68, - 0x91376c36d99995be,0x23100809b9c21fa1, - 0xb58547448ffffb2d,0xabd40a0c2832a78a, - 0xe2e69915b3fff9f9,0x16c90c8f323f516c, - 0x8dd01fad907ffc3b,0xae3da7d97f6792e3, - 0xb1442798f49ffb4a,0x99cd11cfdf41779c, - 0xdd95317f31c7fa1d,0x40405643d711d583, - 0x8a7d3eef7f1cfc52,0x482835ea666b2572, - 0xad1c8eab5ee43b66,0xda3243650005eecf, - 0xd863b256369d4a40,0x90bed43e40076a82, - 0x873e4f75e2224e68,0x5a7744a6e804a291, - 0xa90de3535aaae202,0x711515d0a205cb36, - 0xd3515c2831559a83,0xd5a5b44ca873e03, - 0x8412d9991ed58091,0xe858790afe9486c2, - 0xa5178fff668ae0b6,0x626e974dbe39a872, - 0xce5d73ff402d98e3,0xfb0a3d212dc8128f, - 0x80fa687f881c7f8e,0x7ce66634bc9d0b99, - 0xa139029f6a239f72,0x1c1fffc1ebc44e80, - 0xc987434744ac874e,0xa327ffb266b56220, - 0xfbe9141915d7a922,0x4bf1ff9f0062baa8, - 0x9d71ac8fada6c9b5,0x6f773fc3603db4a9, - 0xc4ce17b399107c22,0xcb550fb4384d21d3, - 0xf6019da07f549b2b,0x7e2a53a146606a48, - 0x99c102844f94e0fb,0x2eda7444cbfc426d, - 0xc0314325637a1939,0xfa911155fefb5308, - 0xf03d93eebc589f88,0x793555ab7eba27ca, - 0x96267c7535b763b5,0x4bc1558b2f3458de, - 0xbbb01b9283253ca2,0x9eb1aaedfb016f16, - 0xea9c227723ee8bcb,0x465e15a979c1cadc, - 0x92a1958a7675175f,0xbfacd89ec191ec9, - 0xb749faed14125d36,0xcef980ec671f667b, - 0xe51c79a85916f484,0x82b7e12780e7401a, - 0x8f31cc0937ae58d2,0xd1b2ecb8b0908810, - 0xb2fe3f0b8599ef07,0x861fa7e6dcb4aa15, - 0xdfbdcece67006ac9,0x67a791e093e1d49a, - 0x8bd6a141006042bd,0xe0c8bb2c5c6d24e0, - 0xaecc49914078536d,0x58fae9f773886e18, - 0xda7f5bf590966848,0xaf39a475506a899e, - 0x888f99797a5e012d,0x6d8406c952429603, - 0xaab37fd7d8f58178,0xc8e5087ba6d33b83, - 0xd5605fcdcf32e1d6,0xfb1e4a9a90880a64, - 0x855c3be0a17fcd26,0x5cf2eea09a55067f, - 0xa6b34ad8c9dfc06f,0xf42faa48c0ea481e, - 0xd0601d8efc57b08b,0xf13b94daf124da26, - 0x823c12795db6ce57,0x76c53d08d6b70858, - 0xa2cb1717b52481ed,0x54768c4b0c64ca6e, - 0xcb7ddcdda26da268,0xa9942f5dcf7dfd09, - 0xfe5d54150b090b02,0xd3f93b35435d7c4c, - 0x9efa548d26e5a6e1,0xc47bc5014a1a6daf, - 0xc6b8e9b0709f109a,0x359ab6419ca1091b, - 0xf867241c8cc6d4c0,0xc30163d203c94b62, - 0x9b407691d7fc44f8,0x79e0de63425dcf1d, - 0xc21094364dfb5636,0x985915fc12f542e4, - 0xf294b943e17a2bc4,0x3e6f5b7b17b2939d, - 0x979cf3ca6cec5b5a,0xa705992ceecf9c42, - 0xbd8430bd08277231,0x50c6ff782a838353, - 0xece53cec4a314ebd,0xa4f8bf5635246428, - 0x940f4613ae5ed136,0x871b7795e136be99, - 0xb913179899f68584,0x28e2557b59846e3f, - 0xe757dd7ec07426e5,0x331aeada2fe589cf, - 0x9096ea6f3848984f,0x3ff0d2c85def7621, - 0xb4bca50b065abe63,0xfed077a756b53a9, - 0xe1ebce4dc7f16dfb,0xd3e8495912c62894, - 0x8d3360f09cf6e4bd,0x64712dd7abbbd95c, - 0xb080392cc4349dec,0xbd8d794d96aacfb3, - 0xdca04777f541c567,0xecf0d7a0fc5583a0, - 0x89e42caaf9491b60,0xf41686c49db57244, - 0xac5d37d5b79b6239,0x311c2875c522ced5, - 0xd77485cb25823ac7,0x7d633293366b828b, - 0x86a8d39ef77164bc,0xae5dff9c02033197, - 0xa8530886b54dbdeb,0xd9f57f830283fdfc, - 0xd267caa862a12d66,0xd072df63c324fd7b, - 0x8380dea93da4bc60,0x4247cb9e59f71e6d, - 0xa46116538d0deb78,0x52d9be85f074e608, - 0xcd795be870516656,0x67902e276c921f8b, - 0x806bd9714632dff6,0xba1cd8a3db53b6, - 0xa086cfcd97bf97f3,0x80e8a40eccd228a4, - 0xc8a883c0fdaf7df0,0x6122cd128006b2cd, - 0xfad2a4b13d1b5d6c,0x796b805720085f81, - 0x9cc3a6eec6311a63,0xcbe3303674053bb0, - 0xc3f490aa77bd60fc,0xbedbfc4411068a9c, - 0xf4f1b4d515acb93b,0xee92fb5515482d44, - 0x991711052d8bf3c5,0x751bdd152d4d1c4a, - 0xbf5cd54678eef0b6,0xd262d45a78a0635d, - 0xef340a98172aace4,0x86fb897116c87c34, - 0x9580869f0e7aac0e,0xd45d35e6ae3d4da0, - 0xbae0a846d2195712,0x8974836059cca109, - 0xe998d258869facd7,0x2bd1a438703fc94b, - 0x91ff83775423cc06,0x7b6306a34627ddcf, - 0xb67f6455292cbf08,0x1a3bc84c17b1d542, - 0xe41f3d6a7377eeca,0x20caba5f1d9e4a93, - 0x8e938662882af53e,0x547eb47b7282ee9c, - 0xb23867fb2a35b28d,0xe99e619a4f23aa43, - 0xdec681f9f4c31f31,0x6405fa00e2ec94d4, - 0x8b3c113c38f9f37e,0xde83bc408dd3dd04, - 0xae0b158b4738705e,0x9624ab50b148d445, - 0xd98ddaee19068c76,0x3badd624dd9b0957, - 0x87f8a8d4cfa417c9,0xe54ca5d70a80e5d6, - 0xa9f6d30a038d1dbc,0x5e9fcf4ccd211f4c, - 0xd47487cc8470652b,0x7647c3200069671f, - 0x84c8d4dfd2c63f3b,0x29ecd9f40041e073, - 0xa5fb0a17c777cf09,0xf468107100525890, - 0xcf79cc9db955c2cc,0x7182148d4066eeb4, - 0x81ac1fe293d599bf,0xc6f14cd848405530, - 0xa21727db38cb002f,0xb8ada00e5a506a7c, - 0xca9cf1d206fdc03b,0xa6d90811f0e4851c, - 0xfd442e4688bd304a,0x908f4a166d1da663, - 0x9e4a9cec15763e2e,0x9a598e4e043287fe, - 0xc5dd44271ad3cdba,0x40eff1e1853f29fd, - 0xf7549530e188c128,0xd12bee59e68ef47c, - 0x9a94dd3e8cf578b9,0x82bb74f8301958ce, - 0xc13a148e3032d6e7,0xe36a52363c1faf01, - 0xf18899b1bc3f8ca1,0xdc44e6c3cb279ac1, - 0x96f5600f15a7b7e5,0x29ab103a5ef8c0b9, - 0xbcb2b812db11a5de,0x7415d448f6b6f0e7, - 0xebdf661791d60f56,0x111b495b3464ad21, - 0x936b9fcebb25c995,0xcab10dd900beec34, - 0xb84687c269ef3bfb,0x3d5d514f40eea742, - 0xe65829b3046b0afa,0xcb4a5a3112a5112, - 0x8ff71a0fe2c2e6dc,0x47f0e785eaba72ab, - 0xb3f4e093db73a093,0x59ed216765690f56, - 0xe0f218b8d25088b8,0x306869c13ec3532c, - 0x8c974f7383725573,0x1e414218c73a13fb, - 0xafbd2350644eeacf,0xe5d1929ef90898fa, - 0xdbac6c247d62a583,0xdf45f746b74abf39, - 0x894bc396ce5da772,0x6b8bba8c328eb783, - 0xab9eb47c81f5114f,0x66ea92f3f326564, - 0xd686619ba27255a2,0xc80a537b0efefebd, - 0x8613fd0145877585,0xbd06742ce95f5f36, - 0xa798fc4196e952e7,0x2c48113823b73704, - 0xd17f3b51fca3a7a0,0xf75a15862ca504c5, - 0x82ef85133de648c4,0x9a984d73dbe722fb, - 0xa3ab66580d5fdaf5,0xc13e60d0d2e0ebba, - 0xcc963fee10b7d1b3,0x318df905079926a8, - 0xffbbcfe994e5c61f,0xfdf17746497f7052, - 0x9fd561f1fd0f9bd3,0xfeb6ea8bedefa633, - 0xc7caba6e7c5382c8,0xfe64a52ee96b8fc0, - 0xf9bd690a1b68637b,0x3dfdce7aa3c673b0, - 0x9c1661a651213e2d,0x6bea10ca65c084e, - 0xc31bfa0fe5698db8,0x486e494fcff30a62, - 0xf3e2f893dec3f126,0x5a89dba3c3efccfa, - 0x986ddb5c6b3a76b7,0xf89629465a75e01c, - 0xbe89523386091465,0xf6bbb397f1135823, - 0xee2ba6c0678b597f,0x746aa07ded582e2c, - 0x94db483840b717ef,0xa8c2a44eb4571cdc, - 0xba121a4650e4ddeb,0x92f34d62616ce413, - 0xe896a0d7e51e1566,0x77b020baf9c81d17, - 0x915e2486ef32cd60,0xace1474dc1d122e, - 0xb5b5ada8aaff80b8,0xd819992132456ba, - 0xe3231912d5bf60e6,0x10e1fff697ed6c69, - 0x8df5efabc5979c8f,0xca8d3ffa1ef463c1, - 0xb1736b96b6fd83b3,0xbd308ff8a6b17cb2, - 0xddd0467c64bce4a0,0xac7cb3f6d05ddbde, - 0x8aa22c0dbef60ee4,0x6bcdf07a423aa96b, - 0xad4ab7112eb3929d,0x86c16c98d2c953c6, - 0xd89d64d57a607744,0xe871c7bf077ba8b7, - 0x87625f056c7c4a8b,0x11471cd764ad4972, - 0xa93af6c6c79b5d2d,0xd598e40d3dd89bcf, - 0xd389b47879823479,0x4aff1d108d4ec2c3, - 0x843610cb4bf160cb,0xcedf722a585139ba, - 0xa54394fe1eedb8fe,0xc2974eb4ee658828, - 0xce947a3da6a9273e,0x733d226229feea32, - 0x811ccc668829b887,0x806357d5a3f525f, - 0xa163ff802a3426a8,0xca07c2dcb0cf26f7, - 0xc9bcff6034c13052,0xfc89b393dd02f0b5, - 0xfc2c3f3841f17c67,0xbbac2078d443ace2, - 0x9d9ba7832936edc0,0xd54b944b84aa4c0d, - 0xc5029163f384a931,0xa9e795e65d4df11, - 0xf64335bcf065d37d,0x4d4617b5ff4a16d5, - 0x99ea0196163fa42e,0x504bced1bf8e4e45, - 0xc06481fb9bcf8d39,0xe45ec2862f71e1d6, - 0xf07da27a82c37088,0x5d767327bb4e5a4c, - 0x964e858c91ba2655,0x3a6a07f8d510f86f, - 0xbbe226efb628afea,0x890489f70a55368b, - 0xeadab0aba3b2dbe5,0x2b45ac74ccea842e, - 0x92c8ae6b464fc96f,0x3b0b8bc90012929d, - 0xb77ada0617e3bbcb,0x9ce6ebb40173744, - 0xe55990879ddcaabd,0xcc420a6a101d0515, - 0x8f57fa54c2a9eab6,0x9fa946824a12232d, - 0xb32df8e9f3546564,0x47939822dc96abf9, - 0xdff9772470297ebd,0x59787e2b93bc56f7, - 0x8bfbea76c619ef36,0x57eb4edb3c55b65a, - 0xaefae51477a06b03,0xede622920b6b23f1, - 0xdab99e59958885c4,0xe95fab368e45eced, - 0x88b402f7fd75539b,0x11dbcb0218ebb414, - 0xaae103b5fcd2a881,0xd652bdc29f26a119, - 0xd59944a37c0752a2,0x4be76d3346f0495f, - 0x857fcae62d8493a5,0x6f70a4400c562ddb, - 0xa6dfbd9fb8e5b88e,0xcb4ccd500f6bb952, - 0xd097ad07a71f26b2,0x7e2000a41346a7a7, - 0x825ecc24c873782f,0x8ed400668c0c28c8, - 0xa2f67f2dfa90563b,0x728900802f0f32fa, - 0xcbb41ef979346bca,0x4f2b40a03ad2ffb9, - 0xfea126b7d78186bc,0xe2f610c84987bfa8, - 0x9f24b832e6b0f436,0xdd9ca7d2df4d7c9, - 0xc6ede63fa05d3143,0x91503d1c79720dbb, - 0xf8a95fcf88747d94,0x75a44c6397ce912a, - 0x9b69dbe1b548ce7c,0xc986afbe3ee11aba, - 0xc24452da229b021b,0xfbe85badce996168, - 0xf2d56790ab41c2a2,0xfae27299423fb9c3, - 0x97c560ba6b0919a5,0xdccd879fc967d41a, - 0xbdb6b8e905cb600f,0x5400e987bbc1c920, - 0xed246723473e3813,0x290123e9aab23b68, - 0x9436c0760c86e30b,0xf9a0b6720aaf6521, - 0xb94470938fa89bce,0xf808e40e8d5b3e69, - 0xe7958cb87392c2c2,0xb60b1d1230b20e04, - 0x90bd77f3483bb9b9,0xb1c6f22b5e6f48c2, - 0xb4ecd5f01a4aa828,0x1e38aeb6360b1af3, - 0xe2280b6c20dd5232,0x25c6da63c38de1b0, - 0x8d590723948a535f,0x579c487e5a38ad0e, - 0xb0af48ec79ace837,0x2d835a9df0c6d851, - 0xdcdb1b2798182244,0xf8e431456cf88e65, - 0x8a08f0f8bf0f156b,0x1b8e9ecb641b58ff, - 0xac8b2d36eed2dac5,0xe272467e3d222f3f, - 0xd7adf884aa879177,0x5b0ed81dcc6abb0f, - 0x86ccbb52ea94baea,0x98e947129fc2b4e9, - 0xa87fea27a539e9a5,0x3f2398d747b36224, - 0xd29fe4b18e88640e,0x8eec7f0d19a03aad, - 0x83a3eeeef9153e89,0x1953cf68300424ac, - 0xa48ceaaab75a8e2b,0x5fa8c3423c052dd7, - 0xcdb02555653131b6,0x3792f412cb06794d, - 0x808e17555f3ebf11,0xe2bbd88bbee40bd0, - 0xa0b19d2ab70e6ed6,0x5b6aceaeae9d0ec4, - 0xc8de047564d20a8b,0xf245825a5a445275, - 0xfb158592be068d2e,0xeed6e2f0f0d56712, - 0x9ced737bb6c4183d,0x55464dd69685606b, - 0xc428d05aa4751e4c,0xaa97e14c3c26b886, - 0xf53304714d9265df,0xd53dd99f4b3066a8, - 0x993fe2c6d07b7fab,0xe546a8038efe4029, - 0xbf8fdb78849a5f96,0xde98520472bdd033, - 0xef73d256a5c0f77c,0x963e66858f6d4440, - 0x95a8637627989aad,0xdde7001379a44aa8, - 0xbb127c53b17ec159,0x5560c018580d5d52, - 0xe9d71b689dde71af,0xaab8f01e6e10b4a6, - 0x9226712162ab070d,0xcab3961304ca70e8, - 0xb6b00d69bb55c8d1,0x3d607b97c5fd0d22, - 0xe45c10c42a2b3b05,0x8cb89a7db77c506a, - 0x8eb98a7a9a5b04e3,0x77f3608e92adb242, - 0xb267ed1940f1c61c,0x55f038b237591ed3, - 0xdf01e85f912e37a3,0x6b6c46dec52f6688, - 0x8b61313bbabce2c6,0x2323ac4b3b3da015, - 0xae397d8aa96c1b77,0xabec975e0a0d081a, - 0xd9c7dced53c72255,0x96e7bd358c904a21, - 0x881cea14545c7575,0x7e50d64177da2e54, - 0xaa242499697392d2,0xdde50bd1d5d0b9e9, - 0xd4ad2dbfc3d07787,0x955e4ec64b44e864, - 0x84ec3c97da624ab4,0xbd5af13bef0b113e, - 0xa6274bbdd0fadd61,0xecb1ad8aeacdd58e, - 0xcfb11ead453994ba,0x67de18eda5814af2, - 0x81ceb32c4b43fcf4,0x80eacf948770ced7, - 0xa2425ff75e14fc31,0xa1258379a94d028d, - 0xcad2f7f5359a3b3e,0x96ee45813a04330, - 0xfd87b5f28300ca0d,0x8bca9d6e188853fc, - 0x9e74d1b791e07e48,0x775ea264cf55347e, - 0xc612062576589dda,0x95364afe032a81a0, - 0xf79687aed3eec551,0x3a83ddbd83f52210, - 0x9abe14cd44753b52,0xc4926a9672793580, - 0xc16d9a0095928a27,0x75b7053c0f178400, - 0xf1c90080baf72cb1,0x5324c68b12dd6800, - 0x971da05074da7bee,0xd3f6fc16ebca8000, - 0xbce5086492111aea,0x88f4bb1ca6bd0000, - 0xec1e4a7db69561a5,0x2b31e9e3d0700000, - 0x9392ee8e921d5d07,0x3aff322e62600000, - 0xb877aa3236a4b449,0x9befeb9fad487c3, - 0xe69594bec44de15b,0x4c2ebe687989a9b4, - 0x901d7cf73ab0acd9,0xf9d37014bf60a11, - 0xb424dc35095cd80f,0x538484c19ef38c95, - 0xe12e13424bb40e13,0x2865a5f206b06fba, - 0x8cbccc096f5088cb,0xf93f87b7442e45d4, - 0xafebff0bcb24aafe,0xf78f69a51539d749, - 0xdbe6fecebdedd5be,0xb573440e5a884d1c, - 0x89705f4136b4a597,0x31680a88f8953031, - 0xabcc77118461cefc,0xfdc20d2b36ba7c3e, - 0xd6bf94d5e57a42bc,0x3d32907604691b4d, - 0x8637bd05af6c69b5,0xa63f9a49c2c1b110, - 0xa7c5ac471b478423,0xfcf80dc33721d54, - 0xd1b71758e219652b,0xd3c36113404ea4a9, - 0x83126e978d4fdf3b,0x645a1cac083126ea, - 0xa3d70a3d70a3d70a,0x3d70a3d70a3d70a4, - 0xcccccccccccccccc,0xcccccccccccccccd, - 0x8000000000000000,0x0, - 0xa000000000000000,0x0, - 0xc800000000000000,0x0, - 0xfa00000000000000,0x0, - 0x9c40000000000000,0x0, - 0xc350000000000000,0x0, - 0xf424000000000000,0x0, - 0x9896800000000000,0x0, - 0xbebc200000000000,0x0, - 0xee6b280000000000,0x0, - 0x9502f90000000000,0x0, - 0xba43b74000000000,0x0, - 0xe8d4a51000000000,0x0, - 0x9184e72a00000000,0x0, - 0xb5e620f480000000,0x0, - 0xe35fa931a0000000,0x0, - 0x8e1bc9bf04000000,0x0, - 0xb1a2bc2ec5000000,0x0, - 0xde0b6b3a76400000,0x0, - 0x8ac7230489e80000,0x0, - 0xad78ebc5ac620000,0x0, - 0xd8d726b7177a8000,0x0, - 0x878678326eac9000,0x0, - 0xa968163f0a57b400,0x0, - 0xd3c21bcecceda100,0x0, - 0x84595161401484a0,0x0, - 0xa56fa5b99019a5c8,0x0, - 0xcecb8f27f4200f3a,0x0, - 0x813f3978f8940984,0x4000000000000000, - 0xa18f07d736b90be5,0x5000000000000000, - 0xc9f2c9cd04674ede,0xa400000000000000, - 0xfc6f7c4045812296,0x4d00000000000000, - 0x9dc5ada82b70b59d,0xf020000000000000, - 0xc5371912364ce305,0x6c28000000000000, - 0xf684df56c3e01bc6,0xc732000000000000, - 0x9a130b963a6c115c,0x3c7f400000000000, - 0xc097ce7bc90715b3,0x4b9f100000000000, - 0xf0bdc21abb48db20,0x1e86d40000000000, - 0x96769950b50d88f4,0x1314448000000000, - 0xbc143fa4e250eb31,0x17d955a000000000, - 0xeb194f8e1ae525fd,0x5dcfab0800000000, - 0x92efd1b8d0cf37be,0x5aa1cae500000000, - 0xb7abc627050305ad,0xf14a3d9e40000000, - 0xe596b7b0c643c719,0x6d9ccd05d0000000, - 0x8f7e32ce7bea5c6f,0xe4820023a2000000, - 0xb35dbf821ae4f38b,0xdda2802c8a800000, - 0xe0352f62a19e306e,0xd50b2037ad200000, - 0x8c213d9da502de45,0x4526f422cc340000, - 0xaf298d050e4395d6,0x9670b12b7f410000, - 0xdaf3f04651d47b4c,0x3c0cdd765f114000, - 0x88d8762bf324cd0f,0xa5880a69fb6ac800, - 0xab0e93b6efee0053,0x8eea0d047a457a00, - 0xd5d238a4abe98068,0x72a4904598d6d880, - 0x85a36366eb71f041,0x47a6da2b7f864750, - 0xa70c3c40a64e6c51,0x999090b65f67d924, - 0xd0cf4b50cfe20765,0xfff4b4e3f741cf6d, - 0x82818f1281ed449f,0xbff8f10e7a8921a4, - 0xa321f2d7226895c7,0xaff72d52192b6a0d, - 0xcbea6f8ceb02bb39,0x9bf4f8a69f764490, - 0xfee50b7025c36a08,0x2f236d04753d5b4, - 0x9f4f2726179a2245,0x1d762422c946590, - 0xc722f0ef9d80aad6,0x424d3ad2b7b97ef5, - 0xf8ebad2b84e0d58b,0xd2e0898765a7deb2, - 0x9b934c3b330c8577,0x63cc55f49f88eb2f, - 0xc2781f49ffcfa6d5,0x3cbf6b71c76b25fb, - 0xf316271c7fc3908a,0x8bef464e3945ef7a, - 0x97edd871cfda3a56,0x97758bf0e3cbb5ac, - 0xbde94e8e43d0c8ec,0x3d52eeed1cbea317, - 0xed63a231d4c4fb27,0x4ca7aaa863ee4bdd, - 0x945e455f24fb1cf8,0x8fe8caa93e74ef6a, - 0xb975d6b6ee39e436,0xb3e2fd538e122b44, - 0xe7d34c64a9c85d44,0x60dbbca87196b616, - 0x90e40fbeea1d3a4a,0xbc8955e946fe31cd, - 0xb51d13aea4a488dd,0x6babab6398bdbe41, - 0xe264589a4dcdab14,0xc696963c7eed2dd1, - 0x8d7eb76070a08aec,0xfc1e1de5cf543ca2, - 0xb0de65388cc8ada8,0x3b25a55f43294bcb, - 0xdd15fe86affad912,0x49ef0eb713f39ebe, - 0x8a2dbf142dfcc7ab,0x6e3569326c784337, - 0xacb92ed9397bf996,0x49c2c37f07965404, - 0xd7e77a8f87daf7fb,0xdc33745ec97be906, - 0x86f0ac99b4e8dafd,0x69a028bb3ded71a3, - 0xa8acd7c0222311bc,0xc40832ea0d68ce0c, - 0xd2d80db02aabd62b,0xf50a3fa490c30190, - 0x83c7088e1aab65db,0x792667c6da79e0fa, - 0xa4b8cab1a1563f52,0x577001b891185938, - 0xcde6fd5e09abcf26,0xed4c0226b55e6f86, - 0x80b05e5ac60b6178,0x544f8158315b05b4, - 0xa0dc75f1778e39d6,0x696361ae3db1c721, - 0xc913936dd571c84c,0x3bc3a19cd1e38e9, - 0xfb5878494ace3a5f,0x4ab48a04065c723, - 0x9d174b2dcec0e47b,0x62eb0d64283f9c76, - 0xc45d1df942711d9a,0x3ba5d0bd324f8394, - 0xf5746577930d6500,0xca8f44ec7ee36479, - 0x9968bf6abbe85f20,0x7e998b13cf4e1ecb, - 0xbfc2ef456ae276e8,0x9e3fedd8c321a67e, - 0xefb3ab16c59b14a2,0xc5cfe94ef3ea101e, - 0x95d04aee3b80ece5,0xbba1f1d158724a12, - 0xbb445da9ca61281f,0x2a8a6e45ae8edc97, - 0xea1575143cf97226,0xf52d09d71a3293bd, - 0x924d692ca61be758,0x593c2626705f9c56, - 0xb6e0c377cfa2e12e,0x6f8b2fb00c77836c, - 0xe498f455c38b997a,0xb6dfb9c0f956447, - 0x8edf98b59a373fec,0x4724bd4189bd5eac, - 0xb2977ee300c50fe7,0x58edec91ec2cb657, - 0xdf3d5e9bc0f653e1,0x2f2967b66737e3ed, - 0x8b865b215899f46c,0xbd79e0d20082ee74, - 0xae67f1e9aec07187,0xecd8590680a3aa11, - 0xda01ee641a708de9,0xe80e6f4820cc9495, - 0x884134fe908658b2,0x3109058d147fdcdd, - 0xaa51823e34a7eede,0xbd4b46f0599fd415, - 0xd4e5e2cdc1d1ea96,0x6c9e18ac7007c91a, - 0x850fadc09923329e,0x3e2cf6bc604ddb0, - 0xa6539930bf6bff45,0x84db8346b786151c, - 0xcfe87f7cef46ff16,0xe612641865679a63, - 0x81f14fae158c5f6e,0x4fcb7e8f3f60c07e, - 0xa26da3999aef7749,0xe3be5e330f38f09d, - 0xcb090c8001ab551c,0x5cadf5bfd3072cc5, - 0xfdcb4fa002162a63,0x73d9732fc7c8f7f6, - 0x9e9f11c4014dda7e,0x2867e7fddcdd9afa, - 0xc646d63501a1511d,0xb281e1fd541501b8, - 0xf7d88bc24209a565,0x1f225a7ca91a4226, - 0x9ae757596946075f,0x3375788de9b06958, - 0xc1a12d2fc3978937,0x52d6b1641c83ae, - 0xf209787bb47d6b84,0xc0678c5dbd23a49a, - 0x9745eb4d50ce6332,0xf840b7ba963646e0, - 0xbd176620a501fbff,0xb650e5a93bc3d898, - 0xec5d3fa8ce427aff,0xa3e51f138ab4cebe, - 0x93ba47c980e98cdf,0xc66f336c36b10137, - 0xb8a8d9bbe123f017,0xb80b0047445d4184, - 0xe6d3102ad96cec1d,0xa60dc059157491e5, - 0x9043ea1ac7e41392,0x87c89837ad68db2f, - 0xb454e4a179dd1877,0x29babe4598c311fb, - 0xe16a1dc9d8545e94,0xf4296dd6fef3d67a, - 0x8ce2529e2734bb1d,0x1899e4a65f58660c, - 0xb01ae745b101e9e4,0x5ec05dcff72e7f8f, - 0xdc21a1171d42645d,0x76707543f4fa1f73, - 0x899504ae72497eba,0x6a06494a791c53a8, - 0xabfa45da0edbde69,0x487db9d17636892, - 0xd6f8d7509292d603,0x45a9d2845d3c42b6, - 0x865b86925b9bc5c2,0xb8a2392ba45a9b2, - 0xa7f26836f282b732,0x8e6cac7768d7141e, - 0xd1ef0244af2364ff,0x3207d795430cd926, - 0x8335616aed761f1f,0x7f44e6bd49e807b8, - 0xa402b9c5a8d3a6e7,0x5f16206c9c6209a6, - 0xcd036837130890a1,0x36dba887c37a8c0f, - 0x802221226be55a64,0xc2494954da2c9789, - 0xa02aa96b06deb0fd,0xf2db9baa10b7bd6c, - 0xc83553c5c8965d3d,0x6f92829494e5acc7, - 0xfa42a8b73abbf48c,0xcb772339ba1f17f9, - 0x9c69a97284b578d7,0xff2a760414536efb, - 0xc38413cf25e2d70d,0xfef5138519684aba, - 0xf46518c2ef5b8cd1,0x7eb258665fc25d69, - 0x98bf2f79d5993802,0xef2f773ffbd97a61, - 0xbeeefb584aff8603,0xaafb550ffacfd8fa, - 0xeeaaba2e5dbf6784,0x95ba2a53f983cf38, - 0x952ab45cfa97a0b2,0xdd945a747bf26183, - 0xba756174393d88df,0x94f971119aeef9e4, - 0xe912b9d1478ceb17,0x7a37cd5601aab85d, - 0x91abb422ccb812ee,0xac62e055c10ab33a, - 0xb616a12b7fe617aa,0x577b986b314d6009, - 0xe39c49765fdf9d94,0xed5a7e85fda0b80b, - 0x8e41ade9fbebc27d,0x14588f13be847307, - 0xb1d219647ae6b31c,0x596eb2d8ae258fc8, - 0xde469fbd99a05fe3,0x6fca5f8ed9aef3bb, - 0x8aec23d680043bee,0x25de7bb9480d5854, - 0xada72ccc20054ae9,0xaf561aa79a10ae6a, - 0xd910f7ff28069da4,0x1b2ba1518094da04, - 0x87aa9aff79042286,0x90fb44d2f05d0842, - 0xa99541bf57452b28,0x353a1607ac744a53, - 0xd3fa922f2d1675f2,0x42889b8997915ce8, - 0x847c9b5d7c2e09b7,0x69956135febada11, - 0xa59bc234db398c25,0x43fab9837e699095, - 0xcf02b2c21207ef2e,0x94f967e45e03f4bb, - 0x8161afb94b44f57d,0x1d1be0eebac278f5, - 0xa1ba1ba79e1632dc,0x6462d92a69731732, - 0xca28a291859bbf93,0x7d7b8f7503cfdcfe, - 0xfcb2cb35e702af78,0x5cda735244c3d43e, - 0x9defbf01b061adab,0x3a0888136afa64a7, - 0xc56baec21c7a1916,0x88aaa1845b8fdd0, - 0xf6c69a72a3989f5b,0x8aad549e57273d45, - 0x9a3c2087a63f6399,0x36ac54e2f678864b, - 0xc0cb28a98fcf3c7f,0x84576a1bb416a7dd, - 0xf0fdf2d3f3c30b9f,0x656d44a2a11c51d5, - 0x969eb7c47859e743,0x9f644ae5a4b1b325, - 0xbc4665b596706114,0x873d5d9f0dde1fee, - 0xeb57ff22fc0c7959,0xa90cb506d155a7ea, - 0x9316ff75dd87cbd8,0x9a7f12442d588f2, - 0xb7dcbf5354e9bece,0xc11ed6d538aeb2f, - 0xe5d3ef282a242e81,0x8f1668c8a86da5fa, - 0x8fa475791a569d10,0xf96e017d694487bc, - 0xb38d92d760ec4455,0x37c981dcc395a9ac, - 0xe070f78d3927556a,0x85bbe253f47b1417, - 0x8c469ab843b89562,0x93956d7478ccec8e, - 0xaf58416654a6babb,0x387ac8d1970027b2, - 0xdb2e51bfe9d0696a,0x6997b05fcc0319e, - 0x88fcf317f22241e2,0x441fece3bdf81f03, - 0xab3c2fddeeaad25a,0xd527e81cad7626c3, - 0xd60b3bd56a5586f1,0x8a71e223d8d3b074, - 0x85c7056562757456,0xf6872d5667844e49, - 0xa738c6bebb12d16c,0xb428f8ac016561db, - 0xd106f86e69d785c7,0xe13336d701beba52, - 0x82a45b450226b39c,0xecc0024661173473, - 0xa34d721642b06084,0x27f002d7f95d0190, - 0xcc20ce9bd35c78a5,0x31ec038df7b441f4, - 0xff290242c83396ce,0x7e67047175a15271, - 0x9f79a169bd203e41,0xf0062c6e984d386, - 0xc75809c42c684dd1,0x52c07b78a3e60868, - 0xf92e0c3537826145,0xa7709a56ccdf8a82, - 0x9bbcc7a142b17ccb,0x88a66076400bb691, - 0xc2abf989935ddbfe,0x6acff893d00ea435, - 0xf356f7ebf83552fe,0x583f6b8c4124d43, - 0x98165af37b2153de,0xc3727a337a8b704a, - 0xbe1bf1b059e9a8d6,0x744f18c0592e4c5c, - 0xeda2ee1c7064130c,0x1162def06f79df73, - 0x9485d4d1c63e8be7,0x8addcb5645ac2ba8, - 0xb9a74a0637ce2ee1,0x6d953e2bd7173692, - 0xe8111c87c5c1ba99,0xc8fa8db6ccdd0437, - 0x910ab1d4db9914a0,0x1d9c9892400a22a2, - 0xb54d5e4a127f59c8,0x2503beb6d00cab4b, - 0xe2a0b5dc971f303a,0x2e44ae64840fd61d, - 0x8da471a9de737e24,0x5ceaecfed289e5d2, - 0xb10d8e1456105dad,0x7425a83e872c5f47, - 0xdd50f1996b947518,0xd12f124e28f77719, - 0x8a5296ffe33cc92f,0x82bd6b70d99aaa6f, - 0xace73cbfdc0bfb7b,0x636cc64d1001550b, - 0xd8210befd30efa5a,0x3c47f7e05401aa4e, - 0x8714a775e3e95c78,0x65acfaec34810a71, - 0xa8d9d1535ce3b396,0x7f1839a741a14d0d, - 0xd31045a8341ca07c,0x1ede48111209a050, - 0x83ea2b892091e44d,0x934aed0aab460432, - 0xa4e4b66b68b65d60,0xf81da84d5617853f, - 0xce1de40642e3f4b9,0x36251260ab9d668e, - 0x80d2ae83e9ce78f3,0xc1d72b7c6b426019, - 0xa1075a24e4421730,0xb24cf65b8612f81f, - 0xc94930ae1d529cfc,0xdee033f26797b627, - 0xfb9b7cd9a4a7443c,0x169840ef017da3b1, - 0x9d412e0806e88aa5,0x8e1f289560ee864e, - 0xc491798a08a2ad4e,0xf1a6f2bab92a27e2, - 0xf5b5d7ec8acb58a2,0xae10af696774b1db, - 0x9991a6f3d6bf1765,0xacca6da1e0a8ef29, - 0xbff610b0cc6edd3f,0x17fd090a58d32af3, - 0xeff394dcff8a948e,0xddfc4b4cef07f5b0, - 0x95f83d0a1fb69cd9,0x4abdaf101564f98e, - 0xbb764c4ca7a4440f,0x9d6d1ad41abe37f1, - 0xea53df5fd18d5513,0x84c86189216dc5ed, - 0x92746b9be2f8552c,0x32fd3cf5b4e49bb4, - 0xb7118682dbb66a77,0x3fbc8c33221dc2a1, - 0xe4d5e82392a40515,0xfabaf3feaa5334a, - 0x8f05b1163ba6832d,0x29cb4d87f2a7400e, - 0xb2c71d5bca9023f8,0x743e20e9ef511012, - 0xdf78e4b2bd342cf6,0x914da9246b255416, - 0x8bab8eefb6409c1a,0x1ad089b6c2f7548e, - 0xae9672aba3d0c320,0xa184ac2473b529b1, - 0xda3c0f568cc4f3e8,0xc9e5d72d90a2741e, - 0x8865899617fb1871,0x7e2fa67c7a658892, - 0xaa7eebfb9df9de8d,0xddbb901b98feeab7, - 0xd51ea6fa85785631,0x552a74227f3ea565, - 0x8533285c936b35de,0xd53a88958f87275f, - 0xa67ff273b8460356,0x8a892abaf368f137, - 0xd01fef10a657842c,0x2d2b7569b0432d85, - 0x8213f56a67f6b29b,0x9c3b29620e29fc73, - 0xa298f2c501f45f42,0x8349f3ba91b47b8f, - 0xcb3f2f7642717713,0x241c70a936219a73, - 0xfe0efb53d30dd4d7,0xed238cd383aa0110, - 0x9ec95d1463e8a506,0xf4363804324a40aa, - 0xc67bb4597ce2ce48,0xb143c6053edcd0d5, - 0xf81aa16fdc1b81da,0xdd94b7868e94050a, - 0x9b10a4e5e9913128,0xca7cf2b4191c8326, - 0xc1d4ce1f63f57d72,0xfd1c2f611f63a3f0, - 0xf24a01a73cf2dccf,0xbc633b39673c8cec, - 0x976e41088617ca01,0xd5be0503e085d813, - 0xbd49d14aa79dbc82,0x4b2d8644d8a74e18, - 0xec9c459d51852ba2,0xddf8e7d60ed1219e, - 0x93e1ab8252f33b45,0xcabb90e5c942b503, - 0xb8da1662e7b00a17,0x3d6a751f3b936243, - 0xe7109bfba19c0c9d,0xcc512670a783ad4, - 0x906a617d450187e2,0x27fb2b80668b24c5, - 0xb484f9dc9641e9da,0xb1f9f660802dedf6, - 0xe1a63853bbd26451,0x5e7873f8a0396973, - 0x8d07e33455637eb2,0xdb0b487b6423e1e8, - 0xb049dc016abc5e5f,0x91ce1a9a3d2cda62, - 0xdc5c5301c56b75f7,0x7641a140cc7810fb, - 0x89b9b3e11b6329ba,0xa9e904c87fcb0a9d, - 0xac2820d9623bf429,0x546345fa9fbdcd44, - 0xd732290fbacaf133,0xa97c177947ad4095, - 0x867f59a9d4bed6c0,0x49ed8eabcccc485d, - 0xa81f301449ee8c70,0x5c68f256bfff5a74, - 0xd226fc195c6a2f8c,0x73832eec6fff3111, - 0x83585d8fd9c25db7,0xc831fd53c5ff7eab, - 0xa42e74f3d032f525,0xba3e7ca8b77f5e55, - 0xcd3a1230c43fb26f,0x28ce1bd2e55f35eb, - 0x80444b5e7aa7cf85,0x7980d163cf5b81b3, - 0xa0555e361951c366,0xd7e105bcc332621f, - 0xc86ab5c39fa63440,0x8dd9472bf3fefaa7, - 0xfa856334878fc150,0xb14f98f6f0feb951, - 0x9c935e00d4b9d8d2,0x6ed1bf9a569f33d3, - 0xc3b8358109e84f07,0xa862f80ec4700c8, - 0xf4a642e14c6262c8,0xcd27bb612758c0fa, - 0x98e7e9cccfbd7dbd,0x8038d51cb897789c, - 0xbf21e44003acdd2c,0xe0470a63e6bd56c3, - 0xeeea5d5004981478,0x1858ccfce06cac74, - 0x95527a5202df0ccb,0xf37801e0c43ebc8, - 0xbaa718e68396cffd,0xd30560258f54e6ba, - 0xe950df20247c83fd,0x47c6b82ef32a2069, - 0x91d28b7416cdd27e,0x4cdc331d57fa5441, - 0xb6472e511c81471d,0xe0133fe4adf8e952, - 0xe3d8f9e563a198e5,0x58180fddd97723a6, - 0x8e679c2f5e44ff8f,0x570f09eaa7ea7648,}; -}; -#endif // SIMDJSON_STATIC_REFLECTION - -extern SIMDJSON_DLLIMPORTEXPORT const uint64_t power_of_five_128[651*2]; -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_NUMBERPARSING_TABLES_H -/* end file simdjson/internal/numberparsing_tables.h */ - -// Precomputed powers of ten from 10^0 to 10^22. These -// can be represented exactly using the double type. -SIMDJSON_DLLIMPORTEXPORT const double simdjson::internal::power_of_ten[] = { - 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, - 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22}; - -/** - * When mapping numbers from decimal to binary, - * we go from w * 10^q to m * 2^p but we have - * 10^q = 5^q * 2^q, so effectively - * we are trying to match - * w * 2^q * 5^q to m * 2^p. Thus the powers of two - * are not a concern since they can be represented - * exactly using the binary notation, only the powers of five - * affect the binary significand. - */ -// The truncated powers of five from 5^-342 all the way to 5^308 -// The mantissa is truncated to 128 bits, and -// never rounded up. Uses about 10KB. -SIMDJSON_DLLIMPORTEXPORT const uint64_t simdjson::internal::power_of_five_128[]= { - 0xeef453d6923bd65a,0x113faa2906a13b3f, - 0x9558b4661b6565f8,0x4ac7ca59a424c507, - 0xbaaee17fa23ebf76,0x5d79bcf00d2df649, - 0xe95a99df8ace6f53,0xf4d82c2c107973dc, - 0x91d8a02bb6c10594,0x79071b9b8a4be869, - 0xb64ec836a47146f9,0x9748e2826cdee284, - 0xe3e27a444d8d98b7,0xfd1b1b2308169b25, - 0x8e6d8c6ab0787f72,0xfe30f0f5e50e20f7, - 0xb208ef855c969f4f,0xbdbd2d335e51a935, - 0xde8b2b66b3bc4723,0xad2c788035e61382, - 0x8b16fb203055ac76,0x4c3bcb5021afcc31, - 0xaddcb9e83c6b1793,0xdf4abe242a1bbf3d, - 0xd953e8624b85dd78,0xd71d6dad34a2af0d, - 0x87d4713d6f33aa6b,0x8672648c40e5ad68, - 0xa9c98d8ccb009506,0x680efdaf511f18c2, - 0xd43bf0effdc0ba48,0x212bd1b2566def2, - 0x84a57695fe98746d,0x14bb630f7604b57, - 0xa5ced43b7e3e9188,0x419ea3bd35385e2d, - 0xcf42894a5dce35ea,0x52064cac828675b9, - 0x818995ce7aa0e1b2,0x7343efebd1940993, - 0xa1ebfb4219491a1f,0x1014ebe6c5f90bf8, - 0xca66fa129f9b60a6,0xd41a26e077774ef6, - 0xfd00b897478238d0,0x8920b098955522b4, - 0x9e20735e8cb16382,0x55b46e5f5d5535b0, - 0xc5a890362fddbc62,0xeb2189f734aa831d, - 0xf712b443bbd52b7b,0xa5e9ec7501d523e4, - 0x9a6bb0aa55653b2d,0x47b233c92125366e, - 0xc1069cd4eabe89f8,0x999ec0bb696e840a, - 0xf148440a256e2c76,0xc00670ea43ca250d, - 0x96cd2a865764dbca,0x380406926a5e5728, - 0xbc807527ed3e12bc,0xc605083704f5ecf2, - 0xeba09271e88d976b,0xf7864a44c633682e, - 0x93445b8731587ea3,0x7ab3ee6afbe0211d, - 0xb8157268fdae9e4c,0x5960ea05bad82964, - 0xe61acf033d1a45df,0x6fb92487298e33bd, - 0x8fd0c16206306bab,0xa5d3b6d479f8e056, - 0xb3c4f1ba87bc8696,0x8f48a4899877186c, - 0xe0b62e2929aba83c,0x331acdabfe94de87, - 0x8c71dcd9ba0b4925,0x9ff0c08b7f1d0b14, - 0xaf8e5410288e1b6f,0x7ecf0ae5ee44dd9, - 0xdb71e91432b1a24a,0xc9e82cd9f69d6150, - 0x892731ac9faf056e,0xbe311c083a225cd2, - 0xab70fe17c79ac6ca,0x6dbd630a48aaf406, - 0xd64d3d9db981787d,0x92cbbccdad5b108, - 0x85f0468293f0eb4e,0x25bbf56008c58ea5, - 0xa76c582338ed2621,0xaf2af2b80af6f24e, - 0xd1476e2c07286faa,0x1af5af660db4aee1, - 0x82cca4db847945ca,0x50d98d9fc890ed4d, - 0xa37fce126597973c,0xe50ff107bab528a0, - 0xcc5fc196fefd7d0c,0x1e53ed49a96272c8, - 0xff77b1fcbebcdc4f,0x25e8e89c13bb0f7a, - 0x9faacf3df73609b1,0x77b191618c54e9ac, - 0xc795830d75038c1d,0xd59df5b9ef6a2417, - 0xf97ae3d0d2446f25,0x4b0573286b44ad1d, - 0x9becce62836ac577,0x4ee367f9430aec32, - 0xc2e801fb244576d5,0x229c41f793cda73f, - 0xf3a20279ed56d48a,0x6b43527578c1110f, - 0x9845418c345644d6,0x830a13896b78aaa9, - 0xbe5691ef416bd60c,0x23cc986bc656d553, - 0xedec366b11c6cb8f,0x2cbfbe86b7ec8aa8, - 0x94b3a202eb1c3f39,0x7bf7d71432f3d6a9, - 0xb9e08a83a5e34f07,0xdaf5ccd93fb0cc53, - 0xe858ad248f5c22c9,0xd1b3400f8f9cff68, - 0x91376c36d99995be,0x23100809b9c21fa1, - 0xb58547448ffffb2d,0xabd40a0c2832a78a, - 0xe2e69915b3fff9f9,0x16c90c8f323f516c, - 0x8dd01fad907ffc3b,0xae3da7d97f6792e3, - 0xb1442798f49ffb4a,0x99cd11cfdf41779c, - 0xdd95317f31c7fa1d,0x40405643d711d583, - 0x8a7d3eef7f1cfc52,0x482835ea666b2572, - 0xad1c8eab5ee43b66,0xda3243650005eecf, - 0xd863b256369d4a40,0x90bed43e40076a82, - 0x873e4f75e2224e68,0x5a7744a6e804a291, - 0xa90de3535aaae202,0x711515d0a205cb36, - 0xd3515c2831559a83,0xd5a5b44ca873e03, - 0x8412d9991ed58091,0xe858790afe9486c2, - 0xa5178fff668ae0b6,0x626e974dbe39a872, - 0xce5d73ff402d98e3,0xfb0a3d212dc8128f, - 0x80fa687f881c7f8e,0x7ce66634bc9d0b99, - 0xa139029f6a239f72,0x1c1fffc1ebc44e80, - 0xc987434744ac874e,0xa327ffb266b56220, - 0xfbe9141915d7a922,0x4bf1ff9f0062baa8, - 0x9d71ac8fada6c9b5,0x6f773fc3603db4a9, - 0xc4ce17b399107c22,0xcb550fb4384d21d3, - 0xf6019da07f549b2b,0x7e2a53a146606a48, - 0x99c102844f94e0fb,0x2eda7444cbfc426d, - 0xc0314325637a1939,0xfa911155fefb5308, - 0xf03d93eebc589f88,0x793555ab7eba27ca, - 0x96267c7535b763b5,0x4bc1558b2f3458de, - 0xbbb01b9283253ca2,0x9eb1aaedfb016f16, - 0xea9c227723ee8bcb,0x465e15a979c1cadc, - 0x92a1958a7675175f,0xbfacd89ec191ec9, - 0xb749faed14125d36,0xcef980ec671f667b, - 0xe51c79a85916f484,0x82b7e12780e7401a, - 0x8f31cc0937ae58d2,0xd1b2ecb8b0908810, - 0xb2fe3f0b8599ef07,0x861fa7e6dcb4aa15, - 0xdfbdcece67006ac9,0x67a791e093e1d49a, - 0x8bd6a141006042bd,0xe0c8bb2c5c6d24e0, - 0xaecc49914078536d,0x58fae9f773886e18, - 0xda7f5bf590966848,0xaf39a475506a899e, - 0x888f99797a5e012d,0x6d8406c952429603, - 0xaab37fd7d8f58178,0xc8e5087ba6d33b83, - 0xd5605fcdcf32e1d6,0xfb1e4a9a90880a64, - 0x855c3be0a17fcd26,0x5cf2eea09a55067f, - 0xa6b34ad8c9dfc06f,0xf42faa48c0ea481e, - 0xd0601d8efc57b08b,0xf13b94daf124da26, - 0x823c12795db6ce57,0x76c53d08d6b70858, - 0xa2cb1717b52481ed,0x54768c4b0c64ca6e, - 0xcb7ddcdda26da268,0xa9942f5dcf7dfd09, - 0xfe5d54150b090b02,0xd3f93b35435d7c4c, - 0x9efa548d26e5a6e1,0xc47bc5014a1a6daf, - 0xc6b8e9b0709f109a,0x359ab6419ca1091b, - 0xf867241c8cc6d4c0,0xc30163d203c94b62, - 0x9b407691d7fc44f8,0x79e0de63425dcf1d, - 0xc21094364dfb5636,0x985915fc12f542e4, - 0xf294b943e17a2bc4,0x3e6f5b7b17b2939d, - 0x979cf3ca6cec5b5a,0xa705992ceecf9c42, - 0xbd8430bd08277231,0x50c6ff782a838353, - 0xece53cec4a314ebd,0xa4f8bf5635246428, - 0x940f4613ae5ed136,0x871b7795e136be99, - 0xb913179899f68584,0x28e2557b59846e3f, - 0xe757dd7ec07426e5,0x331aeada2fe589cf, - 0x9096ea6f3848984f,0x3ff0d2c85def7621, - 0xb4bca50b065abe63,0xfed077a756b53a9, - 0xe1ebce4dc7f16dfb,0xd3e8495912c62894, - 0x8d3360f09cf6e4bd,0x64712dd7abbbd95c, - 0xb080392cc4349dec,0xbd8d794d96aacfb3, - 0xdca04777f541c567,0xecf0d7a0fc5583a0, - 0x89e42caaf9491b60,0xf41686c49db57244, - 0xac5d37d5b79b6239,0x311c2875c522ced5, - 0xd77485cb25823ac7,0x7d633293366b828b, - 0x86a8d39ef77164bc,0xae5dff9c02033197, - 0xa8530886b54dbdeb,0xd9f57f830283fdfc, - 0xd267caa862a12d66,0xd072df63c324fd7b, - 0x8380dea93da4bc60,0x4247cb9e59f71e6d, - 0xa46116538d0deb78,0x52d9be85f074e608, - 0xcd795be870516656,0x67902e276c921f8b, - 0x806bd9714632dff6,0xba1cd8a3db53b6, - 0xa086cfcd97bf97f3,0x80e8a40eccd228a4, - 0xc8a883c0fdaf7df0,0x6122cd128006b2cd, - 0xfad2a4b13d1b5d6c,0x796b805720085f81, - 0x9cc3a6eec6311a63,0xcbe3303674053bb0, - 0xc3f490aa77bd60fc,0xbedbfc4411068a9c, - 0xf4f1b4d515acb93b,0xee92fb5515482d44, - 0x991711052d8bf3c5,0x751bdd152d4d1c4a, - 0xbf5cd54678eef0b6,0xd262d45a78a0635d, - 0xef340a98172aace4,0x86fb897116c87c34, - 0x9580869f0e7aac0e,0xd45d35e6ae3d4da0, - 0xbae0a846d2195712,0x8974836059cca109, - 0xe998d258869facd7,0x2bd1a438703fc94b, - 0x91ff83775423cc06,0x7b6306a34627ddcf, - 0xb67f6455292cbf08,0x1a3bc84c17b1d542, - 0xe41f3d6a7377eeca,0x20caba5f1d9e4a93, - 0x8e938662882af53e,0x547eb47b7282ee9c, - 0xb23867fb2a35b28d,0xe99e619a4f23aa43, - 0xdec681f9f4c31f31,0x6405fa00e2ec94d4, - 0x8b3c113c38f9f37e,0xde83bc408dd3dd04, - 0xae0b158b4738705e,0x9624ab50b148d445, - 0xd98ddaee19068c76,0x3badd624dd9b0957, - 0x87f8a8d4cfa417c9,0xe54ca5d70a80e5d6, - 0xa9f6d30a038d1dbc,0x5e9fcf4ccd211f4c, - 0xd47487cc8470652b,0x7647c3200069671f, - 0x84c8d4dfd2c63f3b,0x29ecd9f40041e073, - 0xa5fb0a17c777cf09,0xf468107100525890, - 0xcf79cc9db955c2cc,0x7182148d4066eeb4, - 0x81ac1fe293d599bf,0xc6f14cd848405530, - 0xa21727db38cb002f,0xb8ada00e5a506a7c, - 0xca9cf1d206fdc03b,0xa6d90811f0e4851c, - 0xfd442e4688bd304a,0x908f4a166d1da663, - 0x9e4a9cec15763e2e,0x9a598e4e043287fe, - 0xc5dd44271ad3cdba,0x40eff1e1853f29fd, - 0xf7549530e188c128,0xd12bee59e68ef47c, - 0x9a94dd3e8cf578b9,0x82bb74f8301958ce, - 0xc13a148e3032d6e7,0xe36a52363c1faf01, - 0xf18899b1bc3f8ca1,0xdc44e6c3cb279ac1, - 0x96f5600f15a7b7e5,0x29ab103a5ef8c0b9, - 0xbcb2b812db11a5de,0x7415d448f6b6f0e7, - 0xebdf661791d60f56,0x111b495b3464ad21, - 0x936b9fcebb25c995,0xcab10dd900beec34, - 0xb84687c269ef3bfb,0x3d5d514f40eea742, - 0xe65829b3046b0afa,0xcb4a5a3112a5112, - 0x8ff71a0fe2c2e6dc,0x47f0e785eaba72ab, - 0xb3f4e093db73a093,0x59ed216765690f56, - 0xe0f218b8d25088b8,0x306869c13ec3532c, - 0x8c974f7383725573,0x1e414218c73a13fb, - 0xafbd2350644eeacf,0xe5d1929ef90898fa, - 0xdbac6c247d62a583,0xdf45f746b74abf39, - 0x894bc396ce5da772,0x6b8bba8c328eb783, - 0xab9eb47c81f5114f,0x66ea92f3f326564, - 0xd686619ba27255a2,0xc80a537b0efefebd, - 0x8613fd0145877585,0xbd06742ce95f5f36, - 0xa798fc4196e952e7,0x2c48113823b73704, - 0xd17f3b51fca3a7a0,0xf75a15862ca504c5, - 0x82ef85133de648c4,0x9a984d73dbe722fb, - 0xa3ab66580d5fdaf5,0xc13e60d0d2e0ebba, - 0xcc963fee10b7d1b3,0x318df905079926a8, - 0xffbbcfe994e5c61f,0xfdf17746497f7052, - 0x9fd561f1fd0f9bd3,0xfeb6ea8bedefa633, - 0xc7caba6e7c5382c8,0xfe64a52ee96b8fc0, - 0xf9bd690a1b68637b,0x3dfdce7aa3c673b0, - 0x9c1661a651213e2d,0x6bea10ca65c084e, - 0xc31bfa0fe5698db8,0x486e494fcff30a62, - 0xf3e2f893dec3f126,0x5a89dba3c3efccfa, - 0x986ddb5c6b3a76b7,0xf89629465a75e01c, - 0xbe89523386091465,0xf6bbb397f1135823, - 0xee2ba6c0678b597f,0x746aa07ded582e2c, - 0x94db483840b717ef,0xa8c2a44eb4571cdc, - 0xba121a4650e4ddeb,0x92f34d62616ce413, - 0xe896a0d7e51e1566,0x77b020baf9c81d17, - 0x915e2486ef32cd60,0xace1474dc1d122e, - 0xb5b5ada8aaff80b8,0xd819992132456ba, - 0xe3231912d5bf60e6,0x10e1fff697ed6c69, - 0x8df5efabc5979c8f,0xca8d3ffa1ef463c1, - 0xb1736b96b6fd83b3,0xbd308ff8a6b17cb2, - 0xddd0467c64bce4a0,0xac7cb3f6d05ddbde, - 0x8aa22c0dbef60ee4,0x6bcdf07a423aa96b, - 0xad4ab7112eb3929d,0x86c16c98d2c953c6, - 0xd89d64d57a607744,0xe871c7bf077ba8b7, - 0x87625f056c7c4a8b,0x11471cd764ad4972, - 0xa93af6c6c79b5d2d,0xd598e40d3dd89bcf, - 0xd389b47879823479,0x4aff1d108d4ec2c3, - 0x843610cb4bf160cb,0xcedf722a585139ba, - 0xa54394fe1eedb8fe,0xc2974eb4ee658828, - 0xce947a3da6a9273e,0x733d226229feea32, - 0x811ccc668829b887,0x806357d5a3f525f, - 0xa163ff802a3426a8,0xca07c2dcb0cf26f7, - 0xc9bcff6034c13052,0xfc89b393dd02f0b5, - 0xfc2c3f3841f17c67,0xbbac2078d443ace2, - 0x9d9ba7832936edc0,0xd54b944b84aa4c0d, - 0xc5029163f384a931,0xa9e795e65d4df11, - 0xf64335bcf065d37d,0x4d4617b5ff4a16d5, - 0x99ea0196163fa42e,0x504bced1bf8e4e45, - 0xc06481fb9bcf8d39,0xe45ec2862f71e1d6, - 0xf07da27a82c37088,0x5d767327bb4e5a4c, - 0x964e858c91ba2655,0x3a6a07f8d510f86f, - 0xbbe226efb628afea,0x890489f70a55368b, - 0xeadab0aba3b2dbe5,0x2b45ac74ccea842e, - 0x92c8ae6b464fc96f,0x3b0b8bc90012929d, - 0xb77ada0617e3bbcb,0x9ce6ebb40173744, - 0xe55990879ddcaabd,0xcc420a6a101d0515, - 0x8f57fa54c2a9eab6,0x9fa946824a12232d, - 0xb32df8e9f3546564,0x47939822dc96abf9, - 0xdff9772470297ebd,0x59787e2b93bc56f7, - 0x8bfbea76c619ef36,0x57eb4edb3c55b65a, - 0xaefae51477a06b03,0xede622920b6b23f1, - 0xdab99e59958885c4,0xe95fab368e45eced, - 0x88b402f7fd75539b,0x11dbcb0218ebb414, - 0xaae103b5fcd2a881,0xd652bdc29f26a119, - 0xd59944a37c0752a2,0x4be76d3346f0495f, - 0x857fcae62d8493a5,0x6f70a4400c562ddb, - 0xa6dfbd9fb8e5b88e,0xcb4ccd500f6bb952, - 0xd097ad07a71f26b2,0x7e2000a41346a7a7, - 0x825ecc24c873782f,0x8ed400668c0c28c8, - 0xa2f67f2dfa90563b,0x728900802f0f32fa, - 0xcbb41ef979346bca,0x4f2b40a03ad2ffb9, - 0xfea126b7d78186bc,0xe2f610c84987bfa8, - 0x9f24b832e6b0f436,0xdd9ca7d2df4d7c9, - 0xc6ede63fa05d3143,0x91503d1c79720dbb, - 0xf8a95fcf88747d94,0x75a44c6397ce912a, - 0x9b69dbe1b548ce7c,0xc986afbe3ee11aba, - 0xc24452da229b021b,0xfbe85badce996168, - 0xf2d56790ab41c2a2,0xfae27299423fb9c3, - 0x97c560ba6b0919a5,0xdccd879fc967d41a, - 0xbdb6b8e905cb600f,0x5400e987bbc1c920, - 0xed246723473e3813,0x290123e9aab23b68, - 0x9436c0760c86e30b,0xf9a0b6720aaf6521, - 0xb94470938fa89bce,0xf808e40e8d5b3e69, - 0xe7958cb87392c2c2,0xb60b1d1230b20e04, - 0x90bd77f3483bb9b9,0xb1c6f22b5e6f48c2, - 0xb4ecd5f01a4aa828,0x1e38aeb6360b1af3, - 0xe2280b6c20dd5232,0x25c6da63c38de1b0, - 0x8d590723948a535f,0x579c487e5a38ad0e, - 0xb0af48ec79ace837,0x2d835a9df0c6d851, - 0xdcdb1b2798182244,0xf8e431456cf88e65, - 0x8a08f0f8bf0f156b,0x1b8e9ecb641b58ff, - 0xac8b2d36eed2dac5,0xe272467e3d222f3f, - 0xd7adf884aa879177,0x5b0ed81dcc6abb0f, - 0x86ccbb52ea94baea,0x98e947129fc2b4e9, - 0xa87fea27a539e9a5,0x3f2398d747b36224, - 0xd29fe4b18e88640e,0x8eec7f0d19a03aad, - 0x83a3eeeef9153e89,0x1953cf68300424ac, - 0xa48ceaaab75a8e2b,0x5fa8c3423c052dd7, - 0xcdb02555653131b6,0x3792f412cb06794d, - 0x808e17555f3ebf11,0xe2bbd88bbee40bd0, - 0xa0b19d2ab70e6ed6,0x5b6aceaeae9d0ec4, - 0xc8de047564d20a8b,0xf245825a5a445275, - 0xfb158592be068d2e,0xeed6e2f0f0d56712, - 0x9ced737bb6c4183d,0x55464dd69685606b, - 0xc428d05aa4751e4c,0xaa97e14c3c26b886, - 0xf53304714d9265df,0xd53dd99f4b3066a8, - 0x993fe2c6d07b7fab,0xe546a8038efe4029, - 0xbf8fdb78849a5f96,0xde98520472bdd033, - 0xef73d256a5c0f77c,0x963e66858f6d4440, - 0x95a8637627989aad,0xdde7001379a44aa8, - 0xbb127c53b17ec159,0x5560c018580d5d52, - 0xe9d71b689dde71af,0xaab8f01e6e10b4a6, - 0x9226712162ab070d,0xcab3961304ca70e8, - 0xb6b00d69bb55c8d1,0x3d607b97c5fd0d22, - 0xe45c10c42a2b3b05,0x8cb89a7db77c506a, - 0x8eb98a7a9a5b04e3,0x77f3608e92adb242, - 0xb267ed1940f1c61c,0x55f038b237591ed3, - 0xdf01e85f912e37a3,0x6b6c46dec52f6688, - 0x8b61313bbabce2c6,0x2323ac4b3b3da015, - 0xae397d8aa96c1b77,0xabec975e0a0d081a, - 0xd9c7dced53c72255,0x96e7bd358c904a21, - 0x881cea14545c7575,0x7e50d64177da2e54, - 0xaa242499697392d2,0xdde50bd1d5d0b9e9, - 0xd4ad2dbfc3d07787,0x955e4ec64b44e864, - 0x84ec3c97da624ab4,0xbd5af13bef0b113e, - 0xa6274bbdd0fadd61,0xecb1ad8aeacdd58e, - 0xcfb11ead453994ba,0x67de18eda5814af2, - 0x81ceb32c4b43fcf4,0x80eacf948770ced7, - 0xa2425ff75e14fc31,0xa1258379a94d028d, - 0xcad2f7f5359a3b3e,0x96ee45813a04330, - 0xfd87b5f28300ca0d,0x8bca9d6e188853fc, - 0x9e74d1b791e07e48,0x775ea264cf55347e, - 0xc612062576589dda,0x95364afe032a81a0, - 0xf79687aed3eec551,0x3a83ddbd83f52210, - 0x9abe14cd44753b52,0xc4926a9672793580, - 0xc16d9a0095928a27,0x75b7053c0f178400, - 0xf1c90080baf72cb1,0x5324c68b12dd6800, - 0x971da05074da7bee,0xd3f6fc16ebca8000, - 0xbce5086492111aea,0x88f4bb1ca6bd0000, - 0xec1e4a7db69561a5,0x2b31e9e3d0700000, - 0x9392ee8e921d5d07,0x3aff322e62600000, - 0xb877aa3236a4b449,0x9befeb9fad487c3, - 0xe69594bec44de15b,0x4c2ebe687989a9b4, - 0x901d7cf73ab0acd9,0xf9d37014bf60a11, - 0xb424dc35095cd80f,0x538484c19ef38c95, - 0xe12e13424bb40e13,0x2865a5f206b06fba, - 0x8cbccc096f5088cb,0xf93f87b7442e45d4, - 0xafebff0bcb24aafe,0xf78f69a51539d749, - 0xdbe6fecebdedd5be,0xb573440e5a884d1c, - 0x89705f4136b4a597,0x31680a88f8953031, - 0xabcc77118461cefc,0xfdc20d2b36ba7c3e, - 0xd6bf94d5e57a42bc,0x3d32907604691b4d, - 0x8637bd05af6c69b5,0xa63f9a49c2c1b110, - 0xa7c5ac471b478423,0xfcf80dc33721d54, - 0xd1b71758e219652b,0xd3c36113404ea4a9, - 0x83126e978d4fdf3b,0x645a1cac083126ea, - 0xa3d70a3d70a3d70a,0x3d70a3d70a3d70a4, - 0xcccccccccccccccc,0xcccccccccccccccd, - 0x8000000000000000,0x0, - 0xa000000000000000,0x0, - 0xc800000000000000,0x0, - 0xfa00000000000000,0x0, - 0x9c40000000000000,0x0, - 0xc350000000000000,0x0, - 0xf424000000000000,0x0, - 0x9896800000000000,0x0, - 0xbebc200000000000,0x0, - 0xee6b280000000000,0x0, - 0x9502f90000000000,0x0, - 0xba43b74000000000,0x0, - 0xe8d4a51000000000,0x0, - 0x9184e72a00000000,0x0, - 0xb5e620f480000000,0x0, - 0xe35fa931a0000000,0x0, - 0x8e1bc9bf04000000,0x0, - 0xb1a2bc2ec5000000,0x0, - 0xde0b6b3a76400000,0x0, - 0x8ac7230489e80000,0x0, - 0xad78ebc5ac620000,0x0, - 0xd8d726b7177a8000,0x0, - 0x878678326eac9000,0x0, - 0xa968163f0a57b400,0x0, - 0xd3c21bcecceda100,0x0, - 0x84595161401484a0,0x0, - 0xa56fa5b99019a5c8,0x0, - 0xcecb8f27f4200f3a,0x0, - 0x813f3978f8940984,0x4000000000000000, - 0xa18f07d736b90be5,0x5000000000000000, - 0xc9f2c9cd04674ede,0xa400000000000000, - 0xfc6f7c4045812296,0x4d00000000000000, - 0x9dc5ada82b70b59d,0xf020000000000000, - 0xc5371912364ce305,0x6c28000000000000, - 0xf684df56c3e01bc6,0xc732000000000000, - 0x9a130b963a6c115c,0x3c7f400000000000, - 0xc097ce7bc90715b3,0x4b9f100000000000, - 0xf0bdc21abb48db20,0x1e86d40000000000, - 0x96769950b50d88f4,0x1314448000000000, - 0xbc143fa4e250eb31,0x17d955a000000000, - 0xeb194f8e1ae525fd,0x5dcfab0800000000, - 0x92efd1b8d0cf37be,0x5aa1cae500000000, - 0xb7abc627050305ad,0xf14a3d9e40000000, - 0xe596b7b0c643c719,0x6d9ccd05d0000000, - 0x8f7e32ce7bea5c6f,0xe4820023a2000000, - 0xb35dbf821ae4f38b,0xdda2802c8a800000, - 0xe0352f62a19e306e,0xd50b2037ad200000, - 0x8c213d9da502de45,0x4526f422cc340000, - 0xaf298d050e4395d6,0x9670b12b7f410000, - 0xdaf3f04651d47b4c,0x3c0cdd765f114000, - 0x88d8762bf324cd0f,0xa5880a69fb6ac800, - 0xab0e93b6efee0053,0x8eea0d047a457a00, - 0xd5d238a4abe98068,0x72a4904598d6d880, - 0x85a36366eb71f041,0x47a6da2b7f864750, - 0xa70c3c40a64e6c51,0x999090b65f67d924, - 0xd0cf4b50cfe20765,0xfff4b4e3f741cf6d, - 0x82818f1281ed449f,0xbff8f10e7a8921a4, - 0xa321f2d7226895c7,0xaff72d52192b6a0d, - 0xcbea6f8ceb02bb39,0x9bf4f8a69f764490, - 0xfee50b7025c36a08,0x2f236d04753d5b4, - 0x9f4f2726179a2245,0x1d762422c946590, - 0xc722f0ef9d80aad6,0x424d3ad2b7b97ef5, - 0xf8ebad2b84e0d58b,0xd2e0898765a7deb2, - 0x9b934c3b330c8577,0x63cc55f49f88eb2f, - 0xc2781f49ffcfa6d5,0x3cbf6b71c76b25fb, - 0xf316271c7fc3908a,0x8bef464e3945ef7a, - 0x97edd871cfda3a56,0x97758bf0e3cbb5ac, - 0xbde94e8e43d0c8ec,0x3d52eeed1cbea317, - 0xed63a231d4c4fb27,0x4ca7aaa863ee4bdd, - 0x945e455f24fb1cf8,0x8fe8caa93e74ef6a, - 0xb975d6b6ee39e436,0xb3e2fd538e122b44, - 0xe7d34c64a9c85d44,0x60dbbca87196b616, - 0x90e40fbeea1d3a4a,0xbc8955e946fe31cd, - 0xb51d13aea4a488dd,0x6babab6398bdbe41, - 0xe264589a4dcdab14,0xc696963c7eed2dd1, - 0x8d7eb76070a08aec,0xfc1e1de5cf543ca2, - 0xb0de65388cc8ada8,0x3b25a55f43294bcb, - 0xdd15fe86affad912,0x49ef0eb713f39ebe, - 0x8a2dbf142dfcc7ab,0x6e3569326c784337, - 0xacb92ed9397bf996,0x49c2c37f07965404, - 0xd7e77a8f87daf7fb,0xdc33745ec97be906, - 0x86f0ac99b4e8dafd,0x69a028bb3ded71a3, - 0xa8acd7c0222311bc,0xc40832ea0d68ce0c, - 0xd2d80db02aabd62b,0xf50a3fa490c30190, - 0x83c7088e1aab65db,0x792667c6da79e0fa, - 0xa4b8cab1a1563f52,0x577001b891185938, - 0xcde6fd5e09abcf26,0xed4c0226b55e6f86, - 0x80b05e5ac60b6178,0x544f8158315b05b4, - 0xa0dc75f1778e39d6,0x696361ae3db1c721, - 0xc913936dd571c84c,0x3bc3a19cd1e38e9, - 0xfb5878494ace3a5f,0x4ab48a04065c723, - 0x9d174b2dcec0e47b,0x62eb0d64283f9c76, - 0xc45d1df942711d9a,0x3ba5d0bd324f8394, - 0xf5746577930d6500,0xca8f44ec7ee36479, - 0x9968bf6abbe85f20,0x7e998b13cf4e1ecb, - 0xbfc2ef456ae276e8,0x9e3fedd8c321a67e, - 0xefb3ab16c59b14a2,0xc5cfe94ef3ea101e, - 0x95d04aee3b80ece5,0xbba1f1d158724a12, - 0xbb445da9ca61281f,0x2a8a6e45ae8edc97, - 0xea1575143cf97226,0xf52d09d71a3293bd, - 0x924d692ca61be758,0x593c2626705f9c56, - 0xb6e0c377cfa2e12e,0x6f8b2fb00c77836c, - 0xe498f455c38b997a,0xb6dfb9c0f956447, - 0x8edf98b59a373fec,0x4724bd4189bd5eac, - 0xb2977ee300c50fe7,0x58edec91ec2cb657, - 0xdf3d5e9bc0f653e1,0x2f2967b66737e3ed, - 0x8b865b215899f46c,0xbd79e0d20082ee74, - 0xae67f1e9aec07187,0xecd8590680a3aa11, - 0xda01ee641a708de9,0xe80e6f4820cc9495, - 0x884134fe908658b2,0x3109058d147fdcdd, - 0xaa51823e34a7eede,0xbd4b46f0599fd415, - 0xd4e5e2cdc1d1ea96,0x6c9e18ac7007c91a, - 0x850fadc09923329e,0x3e2cf6bc604ddb0, - 0xa6539930bf6bff45,0x84db8346b786151c, - 0xcfe87f7cef46ff16,0xe612641865679a63, - 0x81f14fae158c5f6e,0x4fcb7e8f3f60c07e, - 0xa26da3999aef7749,0xe3be5e330f38f09d, - 0xcb090c8001ab551c,0x5cadf5bfd3072cc5, - 0xfdcb4fa002162a63,0x73d9732fc7c8f7f6, - 0x9e9f11c4014dda7e,0x2867e7fddcdd9afa, - 0xc646d63501a1511d,0xb281e1fd541501b8, - 0xf7d88bc24209a565,0x1f225a7ca91a4226, - 0x9ae757596946075f,0x3375788de9b06958, - 0xc1a12d2fc3978937,0x52d6b1641c83ae, - 0xf209787bb47d6b84,0xc0678c5dbd23a49a, - 0x9745eb4d50ce6332,0xf840b7ba963646e0, - 0xbd176620a501fbff,0xb650e5a93bc3d898, - 0xec5d3fa8ce427aff,0xa3e51f138ab4cebe, - 0x93ba47c980e98cdf,0xc66f336c36b10137, - 0xb8a8d9bbe123f017,0xb80b0047445d4184, - 0xe6d3102ad96cec1d,0xa60dc059157491e5, - 0x9043ea1ac7e41392,0x87c89837ad68db2f, - 0xb454e4a179dd1877,0x29babe4598c311fb, - 0xe16a1dc9d8545e94,0xf4296dd6fef3d67a, - 0x8ce2529e2734bb1d,0x1899e4a65f58660c, - 0xb01ae745b101e9e4,0x5ec05dcff72e7f8f, - 0xdc21a1171d42645d,0x76707543f4fa1f73, - 0x899504ae72497eba,0x6a06494a791c53a8, - 0xabfa45da0edbde69,0x487db9d17636892, - 0xd6f8d7509292d603,0x45a9d2845d3c42b6, - 0x865b86925b9bc5c2,0xb8a2392ba45a9b2, - 0xa7f26836f282b732,0x8e6cac7768d7141e, - 0xd1ef0244af2364ff,0x3207d795430cd926, - 0x8335616aed761f1f,0x7f44e6bd49e807b8, - 0xa402b9c5a8d3a6e7,0x5f16206c9c6209a6, - 0xcd036837130890a1,0x36dba887c37a8c0f, - 0x802221226be55a64,0xc2494954da2c9789, - 0xa02aa96b06deb0fd,0xf2db9baa10b7bd6c, - 0xc83553c5c8965d3d,0x6f92829494e5acc7, - 0xfa42a8b73abbf48c,0xcb772339ba1f17f9, - 0x9c69a97284b578d7,0xff2a760414536efb, - 0xc38413cf25e2d70d,0xfef5138519684aba, - 0xf46518c2ef5b8cd1,0x7eb258665fc25d69, - 0x98bf2f79d5993802,0xef2f773ffbd97a61, - 0xbeeefb584aff8603,0xaafb550ffacfd8fa, - 0xeeaaba2e5dbf6784,0x95ba2a53f983cf38, - 0x952ab45cfa97a0b2,0xdd945a747bf26183, - 0xba756174393d88df,0x94f971119aeef9e4, - 0xe912b9d1478ceb17,0x7a37cd5601aab85d, - 0x91abb422ccb812ee,0xac62e055c10ab33a, - 0xb616a12b7fe617aa,0x577b986b314d6009, - 0xe39c49765fdf9d94,0xed5a7e85fda0b80b, - 0x8e41ade9fbebc27d,0x14588f13be847307, - 0xb1d219647ae6b31c,0x596eb2d8ae258fc8, - 0xde469fbd99a05fe3,0x6fca5f8ed9aef3bb, - 0x8aec23d680043bee,0x25de7bb9480d5854, - 0xada72ccc20054ae9,0xaf561aa79a10ae6a, - 0xd910f7ff28069da4,0x1b2ba1518094da04, - 0x87aa9aff79042286,0x90fb44d2f05d0842, - 0xa99541bf57452b28,0x353a1607ac744a53, - 0xd3fa922f2d1675f2,0x42889b8997915ce8, - 0x847c9b5d7c2e09b7,0x69956135febada11, - 0xa59bc234db398c25,0x43fab9837e699095, - 0xcf02b2c21207ef2e,0x94f967e45e03f4bb, - 0x8161afb94b44f57d,0x1d1be0eebac278f5, - 0xa1ba1ba79e1632dc,0x6462d92a69731732, - 0xca28a291859bbf93,0x7d7b8f7503cfdcfe, - 0xfcb2cb35e702af78,0x5cda735244c3d43e, - 0x9defbf01b061adab,0x3a0888136afa64a7, - 0xc56baec21c7a1916,0x88aaa1845b8fdd0, - 0xf6c69a72a3989f5b,0x8aad549e57273d45, - 0x9a3c2087a63f6399,0x36ac54e2f678864b, - 0xc0cb28a98fcf3c7f,0x84576a1bb416a7dd, - 0xf0fdf2d3f3c30b9f,0x656d44a2a11c51d5, - 0x969eb7c47859e743,0x9f644ae5a4b1b325, - 0xbc4665b596706114,0x873d5d9f0dde1fee, - 0xeb57ff22fc0c7959,0xa90cb506d155a7ea, - 0x9316ff75dd87cbd8,0x9a7f12442d588f2, - 0xb7dcbf5354e9bece,0xc11ed6d538aeb2f, - 0xe5d3ef282a242e81,0x8f1668c8a86da5fa, - 0x8fa475791a569d10,0xf96e017d694487bc, - 0xb38d92d760ec4455,0x37c981dcc395a9ac, - 0xe070f78d3927556a,0x85bbe253f47b1417, - 0x8c469ab843b89562,0x93956d7478ccec8e, - 0xaf58416654a6babb,0x387ac8d1970027b2, - 0xdb2e51bfe9d0696a,0x6997b05fcc0319e, - 0x88fcf317f22241e2,0x441fece3bdf81f03, - 0xab3c2fddeeaad25a,0xd527e81cad7626c3, - 0xd60b3bd56a5586f1,0x8a71e223d8d3b074, - 0x85c7056562757456,0xf6872d5667844e49, - 0xa738c6bebb12d16c,0xb428f8ac016561db, - 0xd106f86e69d785c7,0xe13336d701beba52, - 0x82a45b450226b39c,0xecc0024661173473, - 0xa34d721642b06084,0x27f002d7f95d0190, - 0xcc20ce9bd35c78a5,0x31ec038df7b441f4, - 0xff290242c83396ce,0x7e67047175a15271, - 0x9f79a169bd203e41,0xf0062c6e984d386, - 0xc75809c42c684dd1,0x52c07b78a3e60868, - 0xf92e0c3537826145,0xa7709a56ccdf8a82, - 0x9bbcc7a142b17ccb,0x88a66076400bb691, - 0xc2abf989935ddbfe,0x6acff893d00ea435, - 0xf356f7ebf83552fe,0x583f6b8c4124d43, - 0x98165af37b2153de,0xc3727a337a8b704a, - 0xbe1bf1b059e9a8d6,0x744f18c0592e4c5c, - 0xeda2ee1c7064130c,0x1162def06f79df73, - 0x9485d4d1c63e8be7,0x8addcb5645ac2ba8, - 0xb9a74a0637ce2ee1,0x6d953e2bd7173692, - 0xe8111c87c5c1ba99,0xc8fa8db6ccdd0437, - 0x910ab1d4db9914a0,0x1d9c9892400a22a2, - 0xb54d5e4a127f59c8,0x2503beb6d00cab4b, - 0xe2a0b5dc971f303a,0x2e44ae64840fd61d, - 0x8da471a9de737e24,0x5ceaecfed289e5d2, - 0xb10d8e1456105dad,0x7425a83e872c5f47, - 0xdd50f1996b947518,0xd12f124e28f77719, - 0x8a5296ffe33cc92f,0x82bd6b70d99aaa6f, - 0xace73cbfdc0bfb7b,0x636cc64d1001550b, - 0xd8210befd30efa5a,0x3c47f7e05401aa4e, - 0x8714a775e3e95c78,0x65acfaec34810a71, - 0xa8d9d1535ce3b396,0x7f1839a741a14d0d, - 0xd31045a8341ca07c,0x1ede48111209a050, - 0x83ea2b892091e44d,0x934aed0aab460432, - 0xa4e4b66b68b65d60,0xf81da84d5617853f, - 0xce1de40642e3f4b9,0x36251260ab9d668e, - 0x80d2ae83e9ce78f3,0xc1d72b7c6b426019, - 0xa1075a24e4421730,0xb24cf65b8612f81f, - 0xc94930ae1d529cfc,0xdee033f26797b627, - 0xfb9b7cd9a4a7443c,0x169840ef017da3b1, - 0x9d412e0806e88aa5,0x8e1f289560ee864e, - 0xc491798a08a2ad4e,0xf1a6f2bab92a27e2, - 0xf5b5d7ec8acb58a2,0xae10af696774b1db, - 0x9991a6f3d6bf1765,0xacca6da1e0a8ef29, - 0xbff610b0cc6edd3f,0x17fd090a58d32af3, - 0xeff394dcff8a948e,0xddfc4b4cef07f5b0, - 0x95f83d0a1fb69cd9,0x4abdaf101564f98e, - 0xbb764c4ca7a4440f,0x9d6d1ad41abe37f1, - 0xea53df5fd18d5513,0x84c86189216dc5ed, - 0x92746b9be2f8552c,0x32fd3cf5b4e49bb4, - 0xb7118682dbb66a77,0x3fbc8c33221dc2a1, - 0xe4d5e82392a40515,0xfabaf3feaa5334a, - 0x8f05b1163ba6832d,0x29cb4d87f2a7400e, - 0xb2c71d5bca9023f8,0x743e20e9ef511012, - 0xdf78e4b2bd342cf6,0x914da9246b255416, - 0x8bab8eefb6409c1a,0x1ad089b6c2f7548e, - 0xae9672aba3d0c320,0xa184ac2473b529b1, - 0xda3c0f568cc4f3e8,0xc9e5d72d90a2741e, - 0x8865899617fb1871,0x7e2fa67c7a658892, - 0xaa7eebfb9df9de8d,0xddbb901b98feeab7, - 0xd51ea6fa85785631,0x552a74227f3ea565, - 0x8533285c936b35de,0xd53a88958f87275f, - 0xa67ff273b8460356,0x8a892abaf368f137, - 0xd01fef10a657842c,0x2d2b7569b0432d85, - 0x8213f56a67f6b29b,0x9c3b29620e29fc73, - 0xa298f2c501f45f42,0x8349f3ba91b47b8f, - 0xcb3f2f7642717713,0x241c70a936219a73, - 0xfe0efb53d30dd4d7,0xed238cd383aa0110, - 0x9ec95d1463e8a506,0xf4363804324a40aa, - 0xc67bb4597ce2ce48,0xb143c6053edcd0d5, - 0xf81aa16fdc1b81da,0xdd94b7868e94050a, - 0x9b10a4e5e9913128,0xca7cf2b4191c8326, - 0xc1d4ce1f63f57d72,0xfd1c2f611f63a3f0, - 0xf24a01a73cf2dccf,0xbc633b39673c8cec, - 0x976e41088617ca01,0xd5be0503e085d813, - 0xbd49d14aa79dbc82,0x4b2d8644d8a74e18, - 0xec9c459d51852ba2,0xddf8e7d60ed1219e, - 0x93e1ab8252f33b45,0xcabb90e5c942b503, - 0xb8da1662e7b00a17,0x3d6a751f3b936243, - 0xe7109bfba19c0c9d,0xcc512670a783ad4, - 0x906a617d450187e2,0x27fb2b80668b24c5, - 0xb484f9dc9641e9da,0xb1f9f660802dedf6, - 0xe1a63853bbd26451,0x5e7873f8a0396973, - 0x8d07e33455637eb2,0xdb0b487b6423e1e8, - 0xb049dc016abc5e5f,0x91ce1a9a3d2cda62, - 0xdc5c5301c56b75f7,0x7641a140cc7810fb, - 0x89b9b3e11b6329ba,0xa9e904c87fcb0a9d, - 0xac2820d9623bf429,0x546345fa9fbdcd44, - 0xd732290fbacaf133,0xa97c177947ad4095, - 0x867f59a9d4bed6c0,0x49ed8eabcccc485d, - 0xa81f301449ee8c70,0x5c68f256bfff5a74, - 0xd226fc195c6a2f8c,0x73832eec6fff3111, - 0x83585d8fd9c25db7,0xc831fd53c5ff7eab, - 0xa42e74f3d032f525,0xba3e7ca8b77f5e55, - 0xcd3a1230c43fb26f,0x28ce1bd2e55f35eb, - 0x80444b5e7aa7cf85,0x7980d163cf5b81b3, - 0xa0555e361951c366,0xd7e105bcc332621f, - 0xc86ab5c39fa63440,0x8dd9472bf3fefaa7, - 0xfa856334878fc150,0xb14f98f6f0feb951, - 0x9c935e00d4b9d8d2,0x6ed1bf9a569f33d3, - 0xc3b8358109e84f07,0xa862f80ec4700c8, - 0xf4a642e14c6262c8,0xcd27bb612758c0fa, - 0x98e7e9cccfbd7dbd,0x8038d51cb897789c, - 0xbf21e44003acdd2c,0xe0470a63e6bd56c3, - 0xeeea5d5004981478,0x1858ccfce06cac74, - 0x95527a5202df0ccb,0xf37801e0c43ebc8, - 0xbaa718e68396cffd,0xd30560258f54e6ba, - 0xe950df20247c83fd,0x47c6b82ef32a2069, - 0x91d28b7416cdd27e,0x4cdc331d57fa5441, - 0xb6472e511c81471d,0xe0133fe4adf8e952, - 0xe3d8f9e563a198e5,0x58180fddd97723a6, - 0x8e679c2f5e44ff8f,0x570f09eaa7ea7648,}; - - -#endif // SIMDJSON_SRC_NUMBERPARSING_TABLES_CPP -/* end file internal/numberparsing_tables.cpp */ -/* including internal/simdprune_tables.cpp: #include */ -/* begin file internal/simdprune_tables.cpp */ -#ifndef SIMDJSON_SRC_SIMDPRUNE_TABLES_CPP -#define SIMDJSON_SRC_SIMDPRUNE_TABLES_CPP - -/* including simdjson/implementation_detection.h: #include */ -/* begin file simdjson/implementation_detection.h */ -#ifndef SIMDJSON_IMPLEMENTATION_DETECTION_H -#define SIMDJSON_IMPLEMENTATION_DETECTION_H - -/* skipped duplicate #include "simdjson/base.h" */ - -// 0 is reserved, because undefined SIMDJSON_IMPLEMENTATION equals 0 in preprocessor macros. -#define SIMDJSON_IMPLEMENTATION_ID_arm64 1 -#define SIMDJSON_IMPLEMENTATION_ID_fallback 2 -#define SIMDJSON_IMPLEMENTATION_ID_haswell 3 -#define SIMDJSON_IMPLEMENTATION_ID_icelake 4 -#define SIMDJSON_IMPLEMENTATION_ID_ppc64 5 -#define SIMDJSON_IMPLEMENTATION_ID_westmere 6 -#define SIMDJSON_IMPLEMENTATION_ID_lsx 7 -#define SIMDJSON_IMPLEMENTATION_ID_lasx 8 -//#define SIMDJSON_IMPLEMENTATION_ID_rvv 9 -#define SIMDJSON_IMPLEMENTATION_ID_rvv_vls 10 - -#define SIMDJSON_IMPLEMENTATION_ID_FOR(IMPL) SIMDJSON_CAT(SIMDJSON_IMPLEMENTATION_ID_, IMPL) -#define SIMDJSON_IMPLEMENTATION_ID SIMDJSON_IMPLEMENTATION_ID_FOR(SIMDJSON_IMPLEMENTATION) - -#define SIMDJSON_IMPLEMENTATION_IS(IMPL) SIMDJSON_IMPLEMENTATION_ID == SIMDJSON_IMPLEMENTATION_ID_FOR(IMPL) - -// -// First, figure out which implementations can be run. Doing it here makes it so we don't have to worry about the order -// in which we include them. -// - -#ifndef SIMDJSON_IMPLEMENTATION_ARM64 -#define SIMDJSON_IMPLEMENTATION_ARM64 (SIMDJSON_IS_ARM64) -#endif -#if SIMDJSON_IMPLEMENTATION_ARM64 && SIMDJSON_IS_ARM64 -#define SIMDJSON_CAN_ALWAYS_RUN_ARM64 1 -#else -#define SIMDJSON_CAN_ALWAYS_RUN_ARM64 0 -#endif - -// Default Icelake to on if this is x86-64. Even if we're not compiled for it, it could be selected -// at runtime. -#ifndef SIMDJSON_IMPLEMENTATION_ICELAKE -#define SIMDJSON_IMPLEMENTATION_ICELAKE ((SIMDJSON_IS_X86_64) && (SIMDJSON_AVX512_ALLOWED) && (SIMDJSON_COMPILER_SUPPORTS_VBMI2)) -#endif - -#ifdef _MSC_VER -// To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this next line, see -// https://github.com/simdjson/simdjson/issues/1247 -#if ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__)) -#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 1 -#else -#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 0 -#endif - -#else - -#if ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__)) -#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 1 -#else -#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 0 -#endif - -#endif - -// Default Haswell to on if this is x86-64. Even if we're not compiled for it, it could be selected -// at runtime. -#ifndef SIMDJSON_IMPLEMENTATION_HASWELL -#if SIMDJSON_CAN_ALWAYS_RUN_ICELAKE -// if icelake is always available, never enable haswell. -#define SIMDJSON_IMPLEMENTATION_HASWELL 0 -#else -#define SIMDJSON_IMPLEMENTATION_HASWELL SIMDJSON_IS_X86_64 -#endif -#endif -#ifdef _MSC_VER -// To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this next line, see -// https://github.com/simdjson/simdjson/issues/1247 -#if ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__)) -#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 1 -#else -#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 0 -#endif - -#else - -#if ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__)) -#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 1 -#else -#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 0 -#endif - -#endif - -// Default Westmere to on if this is x86-64. -#ifndef SIMDJSON_IMPLEMENTATION_WESTMERE -#if SIMDJSON_CAN_ALWAYS_RUN_ICELAKE || SIMDJSON_CAN_ALWAYS_RUN_HASWELL -// if icelake or haswell are always available, never enable westmere. -#define SIMDJSON_IMPLEMENTATION_WESTMERE 0 -#else -#define SIMDJSON_IMPLEMENTATION_WESTMERE SIMDJSON_IS_X86_64 -#endif -#endif - -#if (SIMDJSON_IMPLEMENTATION_WESTMERE && SIMDJSON_IS_X86_64 && __SSE4_2__ && __PCLMUL__) -#define SIMDJSON_CAN_ALWAYS_RUN_WESTMERE 1 -#else -#define SIMDJSON_CAN_ALWAYS_RUN_WESTMERE 0 -#endif - - -#ifndef SIMDJSON_IMPLEMENTATION_PPC64 -#define SIMDJSON_IMPLEMENTATION_PPC64 (SIMDJSON_IS_PPC64 && SIMDJSON_IS_PPC64_VMX) -#endif -#if SIMDJSON_IMPLEMENTATION_PPC64 && SIMDJSON_IS_PPC64 && SIMDJSON_IS_PPC64_VMX -#define SIMDJSON_CAN_ALWAYS_RUN_PPC64 1 -#else -#define SIMDJSON_CAN_ALWAYS_RUN_PPC64 0 -#endif - -#ifndef SIMDJSON_IMPLEMENTATION_LASX -#define SIMDJSON_IMPLEMENTATION_LASX (SIMDJSON_IS_LSX) -#endif -#define SIMDJSON_CAN_ALWAYS_RUN_LASX (SIMDJSON_IS_LASX) - -#ifndef SIMDJSON_IMPLEMENTATION_LSX -#if SIMDJSON_CAN_ALWAYS_RUN_LASX -#define SIMDJSON_IMPLEMENTATION_LSX 0 -#else -#define SIMDJSON_IMPLEMENTATION_LSX (SIMDJSON_IS_LSX) -#endif -#endif -#define SIMDJSON_CAN_ALWAYS_RUN_LSX (SIMDJSON_IMPLEMENTATION_LSX) - -#define SIMDJSON_CAN_ALWAYS_RUN_RVV_VLS SIMDJSON_IS_RVV_VLS -#ifndef SIMDJSON_IMPLEMENTATION_RVV_VLS -#define SIMDJSON_IMPLEMENTATION_RVV_VLS SIMDJSON_CAN_ALWAYS_RUN_RVV_VLS -#endif - -// Default Fallback to on unless a builtin implementation has already been selected. -#ifndef SIMDJSON_IMPLEMENTATION_FALLBACK -#if SIMDJSON_CAN_ALWAYS_RUN_ARM64 || SIMDJSON_CAN_ALWAYS_RUN_ICELAKE || SIMDJSON_CAN_ALWAYS_RUN_HASWELL || SIMDJSON_CAN_ALWAYS_RUN_WESTMERE || SIMDJSON_CAN_ALWAYS_RUN_PPC64 || SIMDJSON_CAN_ALWAYS_RUN_LSX || SIMDJSON_CAN_ALWAYS_RUN_LASX || SIMDJSON_CAN_ALWAYS_RUN_RVV_VLS -// if anything at all except fallback can always run, then disable fallback. -#define SIMDJSON_IMPLEMENTATION_FALLBACK 0 -#else -#define SIMDJSON_IMPLEMENTATION_FALLBACK 1 -#endif -#endif -#define SIMDJSON_CAN_ALWAYS_RUN_FALLBACK SIMDJSON_IMPLEMENTATION_FALLBACK - -// Determine the best builtin implementation -#ifndef SIMDJSON_BUILTIN_IMPLEMENTATION - -#if SIMDJSON_CAN_ALWAYS_RUN_ICELAKE -#define SIMDJSON_BUILTIN_IMPLEMENTATION icelake -#elif SIMDJSON_CAN_ALWAYS_RUN_HASWELL -#define SIMDJSON_BUILTIN_IMPLEMENTATION haswell -#elif SIMDJSON_CAN_ALWAYS_RUN_WESTMERE -#define SIMDJSON_BUILTIN_IMPLEMENTATION westmere -#elif SIMDJSON_CAN_ALWAYS_RUN_ARM64 -#define SIMDJSON_BUILTIN_IMPLEMENTATION arm64 -#elif SIMDJSON_CAN_ALWAYS_RUN_PPC64 -#define SIMDJSON_BUILTIN_IMPLEMENTATION ppc64 -#elif SIMDJSON_CAN_ALWAYS_RUN_LSX -#define SIMDJSON_BUILTIN_IMPLEMENTATION lsx -#elif SIMDJSON_CAN_ALWAYS_RUN_LASX -#define SIMDJSON_BUILTIN_IMPLEMENTATION lasx -#elif SIMDJSON_CAN_ALWAYS_RUN_RVV_VLS -#define SIMDJSON_BUILTIN_IMPLEMENTATION rvv_vls -#elif SIMDJSON_CAN_ALWAYS_RUN_FALLBACK -#define SIMDJSON_BUILTIN_IMPLEMENTATION fallback -#else -#error "All possible implementations (including fallback) have been disabled! simdjson will not run." -#endif - -#endif // SIMDJSON_BUILTIN_IMPLEMENTATION - -#define SIMDJSON_BUILTIN_IMPLEMENTATION_ID SIMDJSON_IMPLEMENTATION_ID_FOR(SIMDJSON_BUILTIN_IMPLEMENTATION) -#define SIMDJSON_BUILTIN_IMPLEMENTATION_IS(IMPL) SIMDJSON_BUILTIN_IMPLEMENTATION_ID == SIMDJSON_IMPLEMENTATION_ID_FOR(IMPL) - -#endif // SIMDJSON_IMPLEMENTATION_DETECTION_H -/* end file simdjson/implementation_detection.h */ - -#if SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_ICELAKE || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE || SIMDJSON_IMPLEMENTATION_PPC64 || SIMDJSON_IMPLEMENTATION_LSX || SIMDJSON_IMPLEMENTATION_LASX - -#include - -namespace simdjson { // table modified and copied from -namespace internal { // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetTable -SIMDJSON_DLLIMPORTEXPORT const unsigned char BitsSetTable256mul2[256] = { - 0, 2, 2, 4, 2, 4, 4, 6, 2, 4, 4, 6, 4, 6, 6, 8, 2, 4, 4, - 6, 4, 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 2, 4, 4, 6, 4, 6, - 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, - 8, 8, 10, 8, 10, 10, 12, 2, 4, 4, 6, 4, 6, 6, 8, 4, 6, 6, 8, - 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, - 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, 8, - 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 2, 4, 4, 6, 4, - 6, 6, 8, 4, 6, 6, 8, 6, 8, 8, 10, 4, 6, 6, 8, 6, 8, 8, 10, - 6, 8, 8, 10, 8, 10, 10, 12, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, - 10, 8, 10, 10, 12, 6, 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, - 12, 14, 4, 6, 6, 8, 6, 8, 8, 10, 6, 8, 8, 10, 8, 10, 10, 12, 6, - 8, 8, 10, 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 6, 8, 8, 10, - 8, 10, 10, 12, 8, 10, 10, 12, 10, 12, 12, 14, 8, 10, 10, 12, 10, 12, 12, - 14, 10, 12, 12, 14, 12, 14, 14, 16}; - -SIMDJSON_DLLIMPORTEXPORT const uint8_t pshufb_combine_table[272] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, - 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, - 0x0f, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x08, - 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x01, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0x00, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, - 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x08, 0x09, 0x0a, 0x0b, - 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, -}; - -// 256 * 8 bytes = 2kB, easily fits in cache. -SIMDJSON_DLLIMPORTEXPORT const uint64_t thintable_epi8[256] = { - 0x0706050403020100, 0x0007060504030201, 0x0007060504030200, - 0x0000070605040302, 0x0007060504030100, 0x0000070605040301, - 0x0000070605040300, 0x0000000706050403, 0x0007060504020100, - 0x0000070605040201, 0x0000070605040200, 0x0000000706050402, - 0x0000070605040100, 0x0000000706050401, 0x0000000706050400, - 0x0000000007060504, 0x0007060503020100, 0x0000070605030201, - 0x0000070605030200, 0x0000000706050302, 0x0000070605030100, - 0x0000000706050301, 0x0000000706050300, 0x0000000007060503, - 0x0000070605020100, 0x0000000706050201, 0x0000000706050200, - 0x0000000007060502, 0x0000000706050100, 0x0000000007060501, - 0x0000000007060500, 0x0000000000070605, 0x0007060403020100, - 0x0000070604030201, 0x0000070604030200, 0x0000000706040302, - 0x0000070604030100, 0x0000000706040301, 0x0000000706040300, - 0x0000000007060403, 0x0000070604020100, 0x0000000706040201, - 0x0000000706040200, 0x0000000007060402, 0x0000000706040100, - 0x0000000007060401, 0x0000000007060400, 0x0000000000070604, - 0x0000070603020100, 0x0000000706030201, 0x0000000706030200, - 0x0000000007060302, 0x0000000706030100, 0x0000000007060301, - 0x0000000007060300, 0x0000000000070603, 0x0000000706020100, - 0x0000000007060201, 0x0000000007060200, 0x0000000000070602, - 0x0000000007060100, 0x0000000000070601, 0x0000000000070600, - 0x0000000000000706, 0x0007050403020100, 0x0000070504030201, - 0x0000070504030200, 0x0000000705040302, 0x0000070504030100, - 0x0000000705040301, 0x0000000705040300, 0x0000000007050403, - 0x0000070504020100, 0x0000000705040201, 0x0000000705040200, - 0x0000000007050402, 0x0000000705040100, 0x0000000007050401, - 0x0000000007050400, 0x0000000000070504, 0x0000070503020100, - 0x0000000705030201, 0x0000000705030200, 0x0000000007050302, - 0x0000000705030100, 0x0000000007050301, 0x0000000007050300, - 0x0000000000070503, 0x0000000705020100, 0x0000000007050201, - 0x0000000007050200, 0x0000000000070502, 0x0000000007050100, - 0x0000000000070501, 0x0000000000070500, 0x0000000000000705, - 0x0000070403020100, 0x0000000704030201, 0x0000000704030200, - 0x0000000007040302, 0x0000000704030100, 0x0000000007040301, - 0x0000000007040300, 0x0000000000070403, 0x0000000704020100, - 0x0000000007040201, 0x0000000007040200, 0x0000000000070402, - 0x0000000007040100, 0x0000000000070401, 0x0000000000070400, - 0x0000000000000704, 0x0000000703020100, 0x0000000007030201, - 0x0000000007030200, 0x0000000000070302, 0x0000000007030100, - 0x0000000000070301, 0x0000000000070300, 0x0000000000000703, - 0x0000000007020100, 0x0000000000070201, 0x0000000000070200, - 0x0000000000000702, 0x0000000000070100, 0x0000000000000701, - 0x0000000000000700, 0x0000000000000007, 0x0006050403020100, - 0x0000060504030201, 0x0000060504030200, 0x0000000605040302, - 0x0000060504030100, 0x0000000605040301, 0x0000000605040300, - 0x0000000006050403, 0x0000060504020100, 0x0000000605040201, - 0x0000000605040200, 0x0000000006050402, 0x0000000605040100, - 0x0000000006050401, 0x0000000006050400, 0x0000000000060504, - 0x0000060503020100, 0x0000000605030201, 0x0000000605030200, - 0x0000000006050302, 0x0000000605030100, 0x0000000006050301, - 0x0000000006050300, 0x0000000000060503, 0x0000000605020100, - 0x0000000006050201, 0x0000000006050200, 0x0000000000060502, - 0x0000000006050100, 0x0000000000060501, 0x0000000000060500, - 0x0000000000000605, 0x0000060403020100, 0x0000000604030201, - 0x0000000604030200, 0x0000000006040302, 0x0000000604030100, - 0x0000000006040301, 0x0000000006040300, 0x0000000000060403, - 0x0000000604020100, 0x0000000006040201, 0x0000000006040200, - 0x0000000000060402, 0x0000000006040100, 0x0000000000060401, - 0x0000000000060400, 0x0000000000000604, 0x0000000603020100, - 0x0000000006030201, 0x0000000006030200, 0x0000000000060302, - 0x0000000006030100, 0x0000000000060301, 0x0000000000060300, - 0x0000000000000603, 0x0000000006020100, 0x0000000000060201, - 0x0000000000060200, 0x0000000000000602, 0x0000000000060100, - 0x0000000000000601, 0x0000000000000600, 0x0000000000000006, - 0x0000050403020100, 0x0000000504030201, 0x0000000504030200, - 0x0000000005040302, 0x0000000504030100, 0x0000000005040301, - 0x0000000005040300, 0x0000000000050403, 0x0000000504020100, - 0x0000000005040201, 0x0000000005040200, 0x0000000000050402, - 0x0000000005040100, 0x0000000000050401, 0x0000000000050400, - 0x0000000000000504, 0x0000000503020100, 0x0000000005030201, - 0x0000000005030200, 0x0000000000050302, 0x0000000005030100, - 0x0000000000050301, 0x0000000000050300, 0x0000000000000503, - 0x0000000005020100, 0x0000000000050201, 0x0000000000050200, - 0x0000000000000502, 0x0000000000050100, 0x0000000000000501, - 0x0000000000000500, 0x0000000000000005, 0x0000000403020100, - 0x0000000004030201, 0x0000000004030200, 0x0000000000040302, - 0x0000000004030100, 0x0000000000040301, 0x0000000000040300, - 0x0000000000000403, 0x0000000004020100, 0x0000000000040201, - 0x0000000000040200, 0x0000000000000402, 0x0000000000040100, - 0x0000000000000401, 0x0000000000000400, 0x0000000000000004, - 0x0000000003020100, 0x0000000000030201, 0x0000000000030200, - 0x0000000000000302, 0x0000000000030100, 0x0000000000000301, - 0x0000000000000300, 0x0000000000000003, 0x0000000000020100, - 0x0000000000000201, 0x0000000000000200, 0x0000000000000002, - 0x0000000000000100, 0x0000000000000001, 0x0000000000000000, - 0x0000000000000000, -}; //static uint64_t thintable_epi8[256] - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_ICELAKE || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE || SIMDJSON_IMPLEMENTATION_PPC64 || SIMDJSON_IMPLEMENTATION_LSX || SIMDJSON_IMPLEMENTATION_LASX - -#endif // SIMDJSON_SRC_SIMDPRUNE_TABLES_CPP -/* end file internal/simdprune_tables.cpp */ - -/* including simdjson/generic/dependencies.h: #include */ -/* begin file simdjson/generic/dependencies.h */ -#ifdef SIMDJSON_CONDITIONAL_INCLUDE -#error simdjson/generic/dependencies.h must be included before defining SIMDJSON_CONDITIONAL_INCLUDE! -#endif - -#ifndef SIMDJSON_GENERIC_DEPENDENCIES_H -#define SIMDJSON_GENERIC_DEPENDENCIES_H - -// Internal headers needed for generics. -// All includes referencing simdjson headers *not* under simdjson/generic must be here! -// Otherwise, amalgamation will fail. -/* skipped duplicate #include "simdjson/base.h" */ -/* including simdjson/implementation.h: #include "simdjson/implementation.h" */ -/* begin file simdjson/implementation.h */ -#ifndef SIMDJSON_IMPLEMENTATION_H -#define SIMDJSON_IMPLEMENTATION_H - -/* including simdjson/internal/atomic_ptr.h: #include "simdjson/internal/atomic_ptr.h" */ -/* begin file simdjson/internal/atomic_ptr.h */ -#ifndef SIMDJSON_INTERNAL_ATOMIC_PTR_H -#define SIMDJSON_INTERNAL_ATOMIC_PTR_H - -/* skipped duplicate #include "simdjson/base.h" */ -#include - -namespace simdjson { -namespace internal { - -template -class atomic_ptr { -public: - atomic_ptr(T *_ptr) : ptr{_ptr} {} - - operator const T*() const { return ptr.load(); } - const T& operator*() const { return *ptr; } - const T* operator->() const { return ptr.load(); } - - operator T*() { return ptr.load(); } - T& operator*() { return *ptr; } - T* operator->() { return ptr.load(); } - atomic_ptr& operator=(T *_ptr) { ptr = _ptr; return *this; } - -private: - std::atomic ptr; -}; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_ATOMIC_PTR_H -/* end file simdjson/internal/atomic_ptr.h */ -/* including simdjson/internal/dom_parser_implementation.h: #include "simdjson/internal/dom_parser_implementation.h" */ -/* begin file simdjson/internal/dom_parser_implementation.h */ -#ifndef SIMDJSON_INTERNAL_DOM_PARSER_IMPLEMENTATION_H -#define SIMDJSON_INTERNAL_DOM_PARSER_IMPLEMENTATION_H - -/* skipped duplicate #include "simdjson/base.h" */ -/* skipped duplicate #include "simdjson/error.h" */ -#include - -namespace simdjson { - -namespace dom { -class document; -} // namespace dom - -/** -* This enum is used with the dom_parser_implementation::stage1 function. -* 1) The regular mode expects a fully formed JSON document. -* 2) The streaming_partial mode expects a possibly truncated -* input within a stream on JSON documents. -* 3) The stream_final mode allows us to truncate final -* unterminated strings. It is useful in conjunction with streaming_partial. -*/ -enum class stage1_mode { - regular, - streaming_partial, streaming_final, - json_sequence_partial, json_sequence_final, - comma_delimited_partial, comma_delimited_final -}; - -/** - * Returns true if mode == streaming_partial or mode == streaming_final - */ -inline bool is_streaming(stage1_mode mode) { - // performance note: it is probably faster to check that mode is different - // from regular than checking that it is either streaming_partial or streaming_final. - return (mode != stage1_mode::regular); - // return (mode == stage1_mode::streaming_partial || mode == stage1_mode::streaming_final); -} - -namespace internal { - - -/** - * An implementation of simdjson's DOM parser for a particular CPU architecture. - * - * This class is expected to be accessed only by pointer, and never move in memory (though the - * pointer can move). - */ -class dom_parser_implementation { -public: - - /** - * @private For internal implementation use - * - * Run a full JSON parse on a single document (stage1 + stage2). - * - * Guaranteed only to be called when capacity > document length. - * - * Overridden by each implementation. - * - * @param buf The json document to parse. *MUST* be allocated up to len + SIMDJSON_PADDING bytes. - * @param len The length of the json document. - * @return The error code, or SUCCESS if there was no error. - */ - simdjson_warn_unused virtual error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept = 0; - - /** - * @private For internal implementation use - * - * Stage 1 of the document parser. - * - * Guaranteed only to be called when capacity > document length. - * - * Overridden by each implementation. - * - * @param buf The json document to parse. - * @param len The length of the json document. - * @param streaming Whether this is being called by parser::parse_many. - * @return The error code, or SUCCESS if there was no error. - */ - simdjson_warn_unused virtual error_code stage1(const uint8_t *buf, size_t len, stage1_mode streaming) noexcept = 0; - - /** - * @private For internal implementation use - * - * Stage 2 of the document parser. - * - * Called after stage1(). - * - * Overridden by each implementation. - * - * @param doc The document to output to. - * @return The error code, or SUCCESS if there was no error. - */ - simdjson_warn_unused virtual error_code stage2(dom::document &doc) noexcept = 0; - - /** - * @private For internal implementation use - * - * Stage 2 of the document parser for parser::parse_many. - * - * Guaranteed only to be called after stage1(). - * Overridden by each implementation. - * - * @param doc The document to output to. - * @return The error code, SUCCESS if there was no error, or EMPTY if all documents have been parsed. - */ - simdjson_warn_unused virtual error_code stage2_next(dom::document &doc) noexcept = 0; - - /** @private A stage 2 entry point for parser::parse_many: see stage2_next_function. */ - typedef error_code (*stage2_next_function_t)(dom_parser_implementation &impl, dom::document &doc); - - /** - * @private For internal implementation use - * - * Get a function that does what stage2_next() does. dom::document_stream picks - * it once per batch and calls it for each document of the batch, so that the - * choice between the bounded and the unchecked walk costs nothing per document. - * - * Overridden by each implementation. - * - * @param bounded Whether the documents must be bounded to the structural - * indexes of the batch (see _bounded_batch). - */ - virtual stage2_next_function_t stage2_next_function(bool bounded) const noexcept = 0; - - /** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - * - * Overridden by each implementation. - * - * @param str pointer to the beginning of a valid UTF-8 JSON string, must end with an unescaped quote. - * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size. - * @param allow_replacement whether we allow a replacement character when the UTF-8 contains unmatched surrogate pairs. - * @return end of the of the written region (exclusive) or nullptr in case of error. - */ - simdjson_warn_unused virtual uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept = 0; - - /** - * Unescape a NON-valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - * - * Overridden by each implementation. - * - * @param str pointer to the beginning of a possibly invalid UTF-8 JSON string, must end with an unescaped quote. - * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size. - * @return end of the of the written region (exclusive) or nullptr in case of error. - */ - simdjson_warn_unused virtual uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept = 0; - - /** - * Change the capacity of this parser. - * - * The capacity can never exceed SIMDJSON_MAXSIZE_BYTES (e.g., 4 GB) - * and an CAPACITY error is returned if it is attempted. - * - * Generally used for reallocation. - * - * @param capacity The new capacity. - * @param max_depth The new max_depth. - * @return The error code, or SUCCESS if there was no error. - */ - virtual error_code set_capacity(size_t capacity) noexcept = 0; - - /** - * Change the max depth of this parser. - * - * Generally used for reallocation. - * - * @param capacity The new capacity. - * @param max_depth The new max_depth. - * @return The error code, or SUCCESS if there was no error. - */ - virtual error_code set_max_depth(size_t max_depth) noexcept = 0; - - /** - * Deallocate this parser. - */ - virtual ~dom_parser_implementation() = default; - - /** Number of structural indices passed from stage 1 to stage 2 */ - uint32_t n_structural_indexes{0}; - /** Structural indices passed from stage 1 to stage 2 */ - std::unique_ptr structural_indexes{}; - /** Next structural index to parse */ - uint32_t next_structural_index{0}; - - /** - * The largest document this parser can support without reallocating. - * - * @return Current capacity, in bytes. - */ - simdjson_pure simdjson_inline size_t capacity() const noexcept; - - /** - * The maximum level of nested object and arrays supported by this parser. - * - * @return Maximum depth, in bytes. - */ - simdjson_pure simdjson_inline size_t max_depth() const noexcept; - - /** - * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length - * and `max_depth` depth. - * - * @param capacity The new capacity. - * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH. - * @return The error, if there is one. - */ - simdjson_warn_unused inline error_code allocate(size_t capacity, size_t max_depth) noexcept; - - -protected: - /** - * The maximum document length this parser supports. - * - * Buffers are large enough to handle any document up to this length. - */ - size_t _capacity{0}; - - /** - * The maximum depth (number of nested objects and arrays) supported by this parser. - * - * Defaults to DEFAULT_MAX_DEPTH. - */ - size_t _max_depth{0}; - -public: - /** Whether to store big integers as strings instead of returning BIGINT_ERROR */ - bool _number_as_string{false}; - - /** - * Whether the input buffer passed to parse() is *not* padded to len + - * SIMDJSON_PADDING bytes. When true, stage 2 string parsing avoids reading - * past buf+len (it finishes the final, near-the-end bytes from a small padded - * scratch buffer). This is set only by the no-padding DOM parse entry points - * (dom::parser::parse_unpadded); the default padded fast path leaves it false - * and is unaffected. - */ - bool _unpadded{false}; - - /** - * Whether stage 2 must bound the documents of the batch indexed by the last - * stage 1 run to its structural indexes (dom::document_stream sets it, see - * document_stream::run_stage1, and passes it to stage2_next_function). Other - * batches keep the unchecked walk. - */ - bool _bounded_batch{false}; - -protected: - - // Declaring these so that subclasses can use them to implement their constructors. - simdjson_inline dom_parser_implementation() noexcept; - simdjson_inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - simdjson_inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - - simdjson_inline dom_parser_implementation(const dom_parser_implementation &) noexcept = delete; - simdjson_inline dom_parser_implementation &operator=(const dom_parser_implementation &other) noexcept = delete; -}; // class dom_parser_implementation - -simdjson_inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -simdjson_inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -simdjson_inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -simdjson_pure simdjson_inline size_t dom_parser_implementation::capacity() const noexcept { - return _capacity; -} - -simdjson_pure simdjson_inline size_t dom_parser_implementation::max_depth() const noexcept { - return _max_depth; -} - -simdjson_warn_unused -inline error_code dom_parser_implementation::allocate(size_t capacity, size_t max_depth) noexcept { - if (this->max_depth() != max_depth) { - error_code err = set_max_depth(max_depth); - if (err) { return err; } - } - if (_capacity != capacity) { - error_code err = set_capacity(capacity); - if (err) { return err; } - } - return SUCCESS; -} - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/internal/dom_parser_implementation.h */ - -#include - -namespace simdjson { - -/** - * Validate the UTF-8 string. - * - * @param buf the string to validate. - * @param len the length of the string in bytes. - * @return true if the string is valid UTF-8. - */ -simdjson_warn_unused bool validate_utf8(const char * buf, size_t len) noexcept; -/** - * Validate the UTF-8 string. - * - * @param sv the string_view to validate. - * @return true if the string is valid UTF-8. - */ -simdjson_inline simdjson_warn_unused bool validate_utf8(const std::string_view sv) noexcept { - return validate_utf8(sv.data(), sv.size()); -} - -/** - * Validate the UTF-8 string. - * - * @param p the string to validate. - * @return true if the string is valid UTF-8. - */ -simdjson_inline simdjson_warn_unused bool validate_utf8(const std::string& s) noexcept { - return validate_utf8(s.data(), s.size()); -} - -/** - * An implementation of simdjson for a particular CPU architecture. - * - * Also used to maintain the currently active implementation. The active implementation is - * automatically initialized on first use to the most advanced implementation supported by the host. - */ -class implementation { -public: - - /** - * The name of this implementation. - * - * const implementation *impl = simdjson::get_active_implementation(); - * cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl; - * - * @return the name of the implementation, e.g. "haswell", "westmere", "arm64". - */ - virtual std::string name() const { return std::string(_name); } - - /** - * The description of this implementation. - * - * const implementation *impl = simdjson::get_active_implementation(); - * cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl; - * - * @return the description of the implementation, e.g. "Intel/AMD AVX2", "Intel/AMD SSE4.2", "ARM NEON". - */ - virtual std::string description() const { return std::string(_description); } - - /** - * The instruction sets this implementation is compiled against - * and the current CPU match. This function may poll the current CPU/system - * and should therefore not be called too often if performance is a concern. - * - * @return true if the implementation can be safely used on the current system (determined at runtime). - */ - bool supported_by_runtime_system() const; - - /** - * @private For internal implementation use - * - * The instruction sets this implementation is compiled against. - * - * @return a mask of all required `internal::instruction_set::` values. - */ - virtual uint32_t required_instruction_sets() const { return _required_instruction_sets; } - - /** - * @private For internal implementation use - * - * const implementation *impl = simdjson::get_active_implementation(); - * cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl; - * - * @param capacity The largest document that will be passed to the parser. - * @param max_depth The maximum JSON object/array nesting this parser is expected to handle. - * @param dst The place to put the resulting parser implementation. - * @return the error code, or SUCCESS if there was no error. - */ - virtual error_code create_dom_parser_implementation( - size_t capacity, - size_t max_depth, - std::unique_ptr &dst - ) const noexcept = 0; - - /** - * @private For internal implementation use - * - * Minify the input string assuming that it represents a JSON string, does not parse or validate. - * - * Overridden by each implementation. - * - * @param buf the json document to minify. - * @param len the length of the json document. - * @param dst the buffer to write the minified document to. *MUST* be allocated up to len + SIMDJSON_PADDING bytes. - * @param dst_len the number of bytes written. Output only. - * @return the error code, or SUCCESS if there was no error. - */ - simdjson_warn_unused virtual error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept = 0; - - - /** - * Validate the UTF-8 string. - * - * Overridden by each implementation. - * - * @param buf the string to validate. - * @param len the length of the string in bytes. - * @return true if and only if the string is valid UTF-8. - */ - simdjson_warn_unused virtual bool validate_utf8(const char *buf, size_t len) const noexcept = 0; - -protected: - /** @private Construct an implementation with the given name and description. For subclasses. */ - simdjson_inline implementation( - std::string_view name, - std::string_view description, - uint32_t required_instruction_sets - ) : - _name(name), - _description(description), - _required_instruction_sets(required_instruction_sets) - { - } -protected: - ~implementation() = default; - -private: - /** - * The name of this implementation. - */ - std::string_view _name; - - /** - * The description of this implementation. - */ - std::string_view _description; - - /** - * Instruction sets required for this implementation. - */ - const uint32_t _required_instruction_sets; -}; - -/** @private */ -namespace internal { - -/** - * The list of available implementations compiled into simdjson. - */ -class available_implementation_list { -public: - /** Get the list of available implementations compiled into simdjson */ - simdjson_inline available_implementation_list() {} - /** Number of implementations */ - size_t size() const noexcept; - /** STL const begin() iterator */ - const implementation * const *begin() const noexcept; - /** STL const end() iterator */ - const implementation * const *end() const noexcept; - - /** - * Get the implementation with the given name. - * - * Case sensitive. - * - * const implementation *impl = simdjson::get_available_implementations()["westmere"]; - * if (!impl) { exit(1); } - * if (!imp->supported_by_runtime_system()) { exit(1); } - * simdjson::get_active_implementation() = impl; - * - * @param name the implementation to find, e.g. "westmere", "haswell", "arm64" - * @return the implementation, or nullptr if the parse failed. - */ - const implementation * operator[](const std::string_view &name) const noexcept { - for (const implementation * impl : *this) { - if (impl->name() == name) { return impl; } - } - return nullptr; - } - - /** - * Detect the most advanced implementation supported by the current host. - * - * This is used to initialize the implementation on startup. - * - * const implementation *impl = simdjson::available_implementation::detect_best_supported(); - * simdjson::get_active_implementation() = impl; - * - * @return the most advanced supported implementation for the current host, or an - * implementation that returns UNSUPPORTED_ARCHITECTURE if there is no supported - * implementation. Will never return nullptr. - */ - const implementation *detect_best_supported() const noexcept; -}; - -} // namespace internal - -/** - * The list of available implementations compiled into simdjson. - */ -extern SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list& get_available_implementations(); - -/** - * The active implementation. - * - * Automatically initialized on first use to the most advanced implementation supported by this hardware. - */ -extern SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr& get_active_implementation(); - -} // namespace simdjson - -#endif // SIMDJSON_IMPLEMENTATION_H -/* end file simdjson/implementation.h */ -/* skipped duplicate #include "simdjson/implementation_detection.h" */ -/* including simdjson/internal/instruction_set.h: #include "simdjson/internal/instruction_set.h" */ -/* begin file simdjson/internal/instruction_set.h */ -/* From -https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h -Highly modified. - -Copyright (c) 2016- Facebook, Inc (Adam Paszke) -Copyright (c) 2014- Facebook, Inc (Soumith Chintala) -Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) -Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) -Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) -Copyright (c) 2011-2013 NYU (Clement Farabet) -Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, -Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute -(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, -Samy Bengio, Johnny Mariethoz) - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories -America and IDIAP Research Institute nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef SIMDJSON_INTERNAL_INSTRUCTION_SET_H -#define SIMDJSON_INTERNAL_INSTRUCTION_SET_H - -namespace simdjson { -namespace internal { - -enum instruction_set { - DEFAULT = 0x0, - NEON = 0x1, - AVX2 = 0x4, - SSE42 = 0x8, - PCLMULQDQ = 0x10, - BMI1 = 0x20, - BMI2 = 0x40, - ALTIVEC = 0x80, - AVX512F = 0x100, - AVX512DQ = 0x200, - AVX512IFMA = 0x400, - AVX512PF = 0x800, - AVX512ER = 0x1000, - AVX512CD = 0x2000, - AVX512BW = 0x4000, - AVX512VL = 0x8000, - AVX512VBMI2 = 0x10000, - LSX = 0x20000, - LASX = 0x40000, - //RVV = 0x80000, - RVV_VLS = 0x100000, - SVE = 0x200000, - SVE2 = 0x400000, -}; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_INSTRUCTION_SET_H -/* end file simdjson/internal/instruction_set.h */ -/* skipped duplicate #include "simdjson/internal/dom_parser_implementation.h" */ -/* skipped duplicate #include "simdjson/internal/jsoncharutils_tables.h" */ -/* skipped duplicate #include "simdjson/internal/numberparsing_tables.h" */ -/* including simdjson/internal/simdprune_tables.h: #include "simdjson/internal/simdprune_tables.h" */ -/* begin file simdjson/internal/simdprune_tables.h */ -#ifndef SIMDJSON_INTERNAL_SIMDPRUNE_TABLES_H -#define SIMDJSON_INTERNAL_SIMDPRUNE_TABLES_H - -/* skipped duplicate #include "simdjson/base.h" */ - -#include - -namespace simdjson { // table modified and copied from -namespace internal { // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetTable - -extern SIMDJSON_DLLIMPORTEXPORT const unsigned char BitsSetTable256mul2[256]; - -extern SIMDJSON_DLLIMPORTEXPORT const uint8_t pshufb_combine_table[272]; - -// 256 * 8 bytes = 2kB, easily fits in cache. -extern SIMDJSON_DLLIMPORTEXPORT const uint64_t thintable_epi8[256]; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_SIMDPRUNE_TABLES_H -/* end file simdjson/internal/simdprune_tables.h */ -#endif // SIMDJSON_GENERIC_DEPENDENCIES_H -/* end file simdjson/generic/dependencies.h */ -/* including generic/dependencies.h: #include */ -/* begin file generic/dependencies.h */ -#ifdef SIMDJSON_CONDITIONAL_INCLUDE -#error generic/dependencies.h must be included before defining SIMDJSON_CONDITIONAL_INCLUDE! -#endif - -#ifndef SIMDJSON_SRC_GENERIC_DEPENDENCIES_H -#define SIMDJSON_SRC_GENERIC_DEPENDENCIES_H - -/* skipped duplicate #include */ - -#endif // SIMDJSON_SRC_GENERIC_DEPENDENCIES_H -/* end file generic/dependencies.h */ -/* including generic/stage1/dependencies.h: #include */ -/* begin file generic/stage1/dependencies.h */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_DEPENDENCIES_H -#define SIMDJSON_SRC_GENERIC_STAGE1_DEPENDENCIES_H - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_DEPENDENCIES_H -/* end file generic/stage1/dependencies.h */ -/* including generic/stage2/dependencies.h: #include */ -/* begin file generic/stage2/dependencies.h */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_DEPENDENCIES_H -#define SIMDJSON_SRC_GENERIC_STAGE2_DEPENDENCIES_H - -/* including simdjson/dom/document.h: #include */ -/* begin file simdjson/dom/document.h */ -#ifndef SIMDJSON_DOM_DOCUMENT_H -#define SIMDJSON_DOM_DOCUMENT_H - -/* including simdjson/dom/base.h: #include "simdjson/dom/base.h" */ -/* begin file simdjson/dom/base.h */ -#ifndef SIMDJSON_DOM_BASE_H -#define SIMDJSON_DOM_BASE_H - -/* skipped duplicate #include "simdjson/base.h" */ - -namespace simdjson { - -/** - * @brief A DOM API on top of the simdjson parser. - */ -namespace dom { - -/** The default batch size for parser.parse_many() and parser.load_many() */ -static constexpr size_t DEFAULT_BATCH_SIZE = 1000000; -/** - * Some adversary might try to set the batch size to 0 or 1, which might cause problems. - * We set a minimum of 32B since anything else is highly likely to be an error. In practice, - * most users will want a much larger batch size. - * - * All non-negative MINIMAL_BATCH_SIZE values should be 'safe' except that, obviously, no JSON - * document can ever span 0 or 1 byte and that very large values would create memory allocation issues. - */ -static constexpr size_t MINIMAL_BATCH_SIZE = 32; - -/** - * It is wasteful to allocate memory for tiny documents (e.g., 4 bytes). - */ -static constexpr size_t MINIMAL_DOCUMENT_CAPACITY = 32; - -class array; -class document; -class document_stream; -class element; -class key_value_pair; -class object; -class parser; - -#ifdef SIMDJSON_THREADS_ENABLED -struct stage1_worker; -#endif // SIMDJSON_THREADS_ENABLED - -} // namespace dom - -namespace internal { - -template -class string_builder; -class tape_ref; - -} // namespace internal - -} // namespace simdjson - -#endif // SIMDJSON_DOM_BASE_H -/* end file simdjson/dom/base.h */ - -#include - -namespace simdjson { -namespace dom { - -/** - * A parsed JSON document. - * - * This class cannot be copied, only moved, to avoid unintended allocations. - */ -class document { -public: - /** - * Create a document container with zero capacity. - * - * The parser will allocate capacity as needed. - */ - document() noexcept = default; - ~document() noexcept = default; - - /** - * Take another document's buffers. - * - * @param other The document to take. Its capacity is zeroed and it is invalidated. - */ - document(document &&other) noexcept = default; - /** @private */ - document(const document &) = delete; // Disallow copying - /** - * Take another document's buffers. - * - * @param other The document to take. Its capacity is zeroed. - */ - document &operator=(document &&other) noexcept = default; - /** @private */ - document &operator=(const document &) = delete; // Disallow copying - - /** - * Get the root element of this document as a JSON array. - */ - element root() const noexcept; - - /** - * @private Dump the raw tape for debugging. - * - * @param os the stream to output to. - * @return false if the tape is likely wrong (e.g., you did not parse a valid JSON). - */ - bool dump_raw_tape(std::ostream &os) const noexcept; - - /** @private Structural values. */ - std::unique_ptr tape{}; - - /** @private String values. - * - * Should be at least byte_capacity. - */ - std::unique_ptr string_buf{}; - /** @private Allocate memory to support - * input JSON documents of up to len bytes. - * - * When calling this function, you lose - * all the data. - * - * The memory allocation is strict: you - * can you use this function to increase - * or lower the amount of allocated memory. - * Passing zero clears the memory. - */ - error_code allocate(size_t len) noexcept; - /** @private Capacity in bytes, in terms - * of how many bytes of input JSON we can - * support. - */ - size_t capacity() const noexcept; - - -private: - size_t allocated_capacity{0}; - friend class parser; -}; // class document - -} // namespace dom -} // namespace simdjson - -#endif // SIMDJSON_DOM_DOCUMENT_H -/* end file simdjson/dom/document.h */ -/* including simdjson/internal/tape_type.h: #include */ -/* begin file simdjson/internal/tape_type.h */ -#ifndef SIMDJSON_INTERNAL_TAPE_TYPE_H -#define SIMDJSON_INTERNAL_TAPE_TYPE_H - -namespace simdjson { -namespace internal { - -/** - * The possible types in the tape. - */ -enum class tape_type { - ROOT = 'r', - START_ARRAY = '[', - START_OBJECT = '{', - END_ARRAY = ']', - END_OBJECT = '}', - STRING = '"', - INT64 = 'l', - UINT64 = 'u', - DOUBLE = 'd', - TRUE_VALUE = 't', - FALSE_VALUE = 'f', - NULL_VALUE = 'n', - BIGINT = 'Z' // Big integer stored as string in string buffer -}; // enum class tape_type - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_TAPE_TYPE_H -/* end file simdjson/internal/tape_type.h */ - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_DEPENDENCIES_H -/* end file generic/stage2/dependencies.h */ - -/* including implementation.cpp: #include */ -/* begin file implementation.cpp */ -#ifndef SIMDJSON_SRC_IMPLEMENTATION_CPP -#define SIMDJSON_SRC_IMPLEMENTATION_CPP - -/* skipped duplicate #include */ -/* skipped duplicate #include */ -/* skipped duplicate #include */ -/* including internal/isadetection.h: #include */ -/* begin file internal/isadetection.h */ -/* From -https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h -Highly modified. - -Copyright (c) 2016- Facebook, Inc (Adam Paszke) -Copyright (c) 2014- Facebook, Inc (Soumith Chintala) -Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) -Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) -Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) -Copyright (c) 2011-2013 NYU (Clement Farabet) -Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, -Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute -(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, -Samy Bengio, Johnny Mariethoz) - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories -America and IDIAP Research Institute nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef SIMDJSON_INTERNAL_ISADETECTION_H -#define SIMDJSON_INTERNAL_ISADETECTION_H - -/* skipped duplicate #include "simdjson/internal/instruction_set.h" */ - -#include -#include -#if defined(_MSC_VER) -#include -#elif (defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)) || defined(__FILC__) -#include -#endif -#if defined(__loongarch__) && defined(__linux__) - #include -#endif -#if defined(__aarch64__) && defined(__linux__) - #include -#endif -#if (defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) && defined(_WIN32) && !defined(_WINDOWS_) -// We avoid including (macro pollution); this matches the -// declaration in the Windows SDK (BOOL WINAPI IsProcessorFeaturePresent(DWORD)). -extern "C" __declspec(dllimport) int __stdcall IsProcessorFeaturePresent(unsigned long ProcessorFeature); -#endif - -#ifdef __FILC__ -#include -#endif - -namespace simdjson { -namespace internal { - -#if defined(__PPC64__) - -static inline uint32_t detect_supported_architectures() { - return instruction_set::ALTIVEC; -} - -#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) - -#if defined(__linux__) -// The kernel advertises SVE in AT_HWCAP and SVE2 in AT_HWCAP2. Older -// headers may not define these constants, so we provide the kernel's values -// (we deliberately do not include , which is not available on -// all toolchains, e.g., musl without linux-headers). -#ifndef AT_HWCAP2 -#define AT_HWCAP2 26 -#endif -#ifndef HWCAP_SVE -#define HWCAP_SVE (1 << 22) -#endif -#ifndef HWCAP2_SVE2 -#define HWCAP2_SVE2 (1 << 1) -#endif -#endif // __linux__ - -#if defined(_WIN32) -// Only recent Windows SDKs define these processor features. -#ifndef PF_ARM_SVE_INSTRUCTIONS_AVAILABLE -#define PF_ARM_SVE_INSTRUCTIONS_AVAILABLE 46 -#endif -#ifndef PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE -#define PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE 47 -#endif -#endif // _WIN32 - -static inline uint32_t detect_supported_architectures() { - // NEON is mandatory on AArch64. - uint32_t host_isa = instruction_set::NEON; -#if defined(__linux__) - unsigned long hwcap = getauxval(AT_HWCAP); - unsigned long hwcap2 = getauxval(AT_HWCAP2); - if (hwcap & HWCAP_SVE) { - host_isa |= instruction_set::SVE; - // We only claim SVE2 when SVE is also present. Before Linux 6.14, the - // kernel set HWCAP2_SVE2 on processors implementing SME(2) but not SVE, - // because SVE2 instructions are available in streaming mode. Our SVE2 - // code runs in non-streaming mode and needs actual SVE. - if (hwcap2 & HWCAP2_SVE2) { - host_isa |= instruction_set::SVE2; - } - } -#elif defined(_WIN32) - if (IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE)) { - host_isa |= instruction_set::SVE; - // As on Linux, require SVE before claiming SVE2. - if (IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE)) { - host_isa |= instruction_set::SVE2; - } - } -#endif - // On other systems (e.g., macOS, where Apple Silicon has no SVE), we only - // report NEON. - return host_isa; -} - -#elif defined(__x86_64__) || defined(_M_AMD64) // x64 - - -namespace { -// Can be found on Intel ISA Reference for CPUID -constexpr uint32_t cpuid_avx2_bit = 1 << 5; ///< @private Bit 5 of EBX for EAX=0x7 -constexpr uint32_t cpuid_bmi1_bit = 1 << 3; ///< @private bit 3 of EBX for EAX=0x7 -constexpr uint32_t cpuid_bmi2_bit = 1 << 8; ///< @private bit 8 of EBX for EAX=0x7 -constexpr uint32_t cpuid_avx512f_bit = 1 << 16; ///< @private bit 16 of EBX for EAX=0x7 -constexpr uint32_t cpuid_avx512dq_bit = 1 << 17; ///< @private bit 17 of EBX for EAX=0x7 -constexpr uint32_t cpuid_avx512ifma_bit = 1 << 21; ///< @private bit 21 of EBX for EAX=0x7 -constexpr uint32_t cpuid_avx512pf_bit = 1 << 26; ///< @private bit 26 of EBX for EAX=0x7 -constexpr uint32_t cpuid_avx512er_bit = 1 << 27; ///< @private bit 27 of EBX for EAX=0x7 -constexpr uint32_t cpuid_avx512cd_bit = 1 << 28; ///< @private bit 28 of EBX for EAX=0x7 -constexpr uint32_t cpuid_avx512bw_bit = 1 << 30; ///< @private bit 30 of EBX for EAX=0x7 -constexpr uint32_t cpuid_avx512vl_bit = 1U << 31; ///< @private bit 31 of EBX for EAX=0x7 -constexpr uint32_t cpuid_avx512vbmi2_bit = 1 << 6; ///< @private bit 6 of ECX for EAX=0x7 -constexpr uint64_t cpuid_avx256_saved = uint64_t(1) << 2; ///< @private bit 2 = AVX -constexpr uint64_t cpuid_avx512_saved = uint64_t(7) << 5; ///< @private bits 5,6,7 = opmask, ZMM_hi256, hi16_ZMM -constexpr uint32_t cpuid_sse42_bit = 1 << 20; ///< @private bit 20 of ECX for EAX=0x1 -constexpr uint32_t cpuid_osxsave = (uint32_t(1) << 26) | (uint32_t(1) << 27); ///< @private bits 26+27 of ECX for EAX=0x1 -constexpr uint32_t cpuid_pclmulqdq_bit = 1 << 1; ///< @private bit 1 of ECX for EAX=0x1 -} - - - -static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, - uint32_t *edx) { -#if defined(_MSC_VER) - int cpu_info[4]; - __cpuidex(cpu_info, *eax, *ecx); - *eax = cpu_info[0]; - *ebx = cpu_info[1]; - *ecx = cpu_info[2]; - *edx = cpu_info[3]; -#elif (defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)) || defined(__FILC__) - uint32_t level = *eax; - __get_cpuid(level, eax, ebx, ecx, edx); -#else - uint32_t a = *eax, b, c = *ecx, d; - asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d)); - *eax = a; - *ebx = b; - *ecx = c; - *edx = d; -#endif -} - - -static inline uint64_t xgetbv() { -#if defined(_MSC_VER) - return _xgetbv(0); -#elif defined(__FILC__) - return zxgetbv(); -#else - uint32_t xcr0_lo, xcr0_hi; - asm volatile("xgetbv\n\t" : "=a" (xcr0_lo), "=d" (xcr0_hi) : "c" (0)); - return xcr0_lo | (uint64_t(xcr0_hi) << 32); -#endif -} - -static inline uint32_t detect_supported_architectures() { - uint32_t eax, ebx, ecx, edx; - uint32_t host_isa = 0x0; - - // EBX for EAX=0x1 - eax = 0x1; - ecx = 0x0; - cpuid(&eax, &ebx, &ecx, &edx); - - if (ecx & cpuid_sse42_bit) { - host_isa |= instruction_set::SSE42; - } else { - return host_isa; // everything after is redundant - } - - if (ecx & cpuid_pclmulqdq_bit) { - host_isa |= instruction_set::PCLMULQDQ; - } - - - if ((ecx & cpuid_osxsave) != cpuid_osxsave) { - return host_isa; - } - - // xgetbv for checking if the OS saves registers - uint64_t xcr0 = xgetbv(); - - if ((xcr0 & cpuid_avx256_saved) == 0) { - return host_isa; - } - - // ECX for EAX=0x7 - eax = 0x7; - ecx = 0x0; - cpuid(&eax, &ebx, &ecx, &edx); - if (ebx & cpuid_avx2_bit) { - host_isa |= instruction_set::AVX2; - } - if (ebx & cpuid_bmi1_bit) { - host_isa |= instruction_set::BMI1; - } - - if (ebx & cpuid_bmi2_bit) { - host_isa |= instruction_set::BMI2; - } - - if (!((xcr0 & cpuid_avx512_saved) == cpuid_avx512_saved)) { - return host_isa; - } - - if (ebx & cpuid_avx512f_bit) { - host_isa |= instruction_set::AVX512F; - } - - if (ebx & cpuid_avx512dq_bit) { - host_isa |= instruction_set::AVX512DQ; - } - - if (ebx & cpuid_avx512ifma_bit) { - host_isa |= instruction_set::AVX512IFMA; - } - - if (ebx & cpuid_avx512pf_bit) { - host_isa |= instruction_set::AVX512PF; - } - - if (ebx & cpuid_avx512er_bit) { - host_isa |= instruction_set::AVX512ER; - } - - if (ebx & cpuid_avx512cd_bit) { - host_isa |= instruction_set::AVX512CD; - } - - if (ebx & cpuid_avx512bw_bit) { - host_isa |= instruction_set::AVX512BW; - } - - if (ebx & cpuid_avx512vl_bit) { - host_isa |= instruction_set::AVX512VL; - } - - if (ecx & cpuid_avx512vbmi2_bit) { - host_isa |= instruction_set::AVX512VBMI2; - } - - return host_isa; -} - -#elif defined(__loongarch__) - -static inline uint32_t detect_supported_architectures() { - uint32_t host_isa = instruction_set::DEFAULT; - #if defined(__linux__) - uint64_t hwcap = 0; - hwcap = getauxval(AT_HWCAP); - if (hwcap & HWCAP_LOONGARCH_LSX) { - host_isa |= instruction_set::LSX; - } - if (hwcap & HWCAP_LOONGARCH_LASX) { - host_isa |= instruction_set::LASX; - } - #endif - return host_isa; -} - -#elif SIMDJSON_IS_RISCV64 - -static inline uint32_t detect_supported_architectures() { - uint32_t host_isa = instruction_set::DEFAULT; -#if SIMDJSON_IS_RVV_VLS - host_isa |= instruction_set::RVV_VLS; -#endif - return host_isa; -} - -#else // fallback - - -static inline uint32_t detect_supported_architectures() { - return instruction_set::DEFAULT; -} - - -#endif // end SIMD extension detection code - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_ISADETECTION_H -/* end file internal/isadetection.h */ - -#include -#include - -namespace simdjson { - -bool implementation::supported_by_runtime_system() const { - uint32_t required_instruction_sets = this->required_instruction_sets(); - uint32_t supported_instruction_sets = internal::detect_supported_architectures(); - return ((supported_instruction_sets & required_instruction_sets) == required_instruction_sets); -} - -} // namespace simdjson - -/* defining SIMDJSON_CONDITIONAL_INCLUDE */ -#define SIMDJSON_CONDITIONAL_INCLUDE - -#if SIMDJSON_IMPLEMENTATION_ARM64 -/* including simdjson/arm64/implementation.h: #include */ -/* begin file simdjson/arm64/implementation.h */ -#ifndef SIMDJSON_ARM64_IMPLEMENTATION_H -#define SIMDJSON_ARM64_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation("arm64", "ARM NEON", internal::instruction_set::NEON) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_ARM64_IMPLEMENTATION_H -/* end file simdjson/arm64/implementation.h */ -namespace simdjson { -namespace internal { -static const arm64::implementation* get_arm64_singleton() { - static const arm64::implementation arm64_singleton{}; - return &arm64_singleton; -} -} // namespace internal -} // namespace simdjson -#endif // SIMDJSON_IMPLEMENTATION_ARM64 - -#if SIMDJSON_IMPLEMENTATION_FALLBACK -/* including simdjson/fallback/implementation.h: #include */ -/* begin file simdjson/fallback/implementation.h */ -#ifndef SIMDJSON_FALLBACK_IMPLEMENTATION_H -#define SIMDJSON_FALLBACK_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation( - "fallback", - "Generic fallback implementation", - 0 - ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_FALLBACK_IMPLEMENTATION_H -/* end file simdjson/fallback/implementation.h */ -namespace simdjson { -namespace internal { -static const fallback::implementation* get_fallback_singleton() { - static const fallback::implementation fallback_singleton{}; - return &fallback_singleton; -} -} // namespace internal -} // namespace simdjson -#endif // SIMDJSON_IMPLEMENTATION_FALLBACK - - -#if SIMDJSON_IMPLEMENTATION_HASWELL -/* including simdjson/haswell/implementation.h: #include */ -/* begin file simdjson/haswell/implementation.h */ -#ifndef SIMDJSON_HASWELL_IMPLEMENTATION_H -#define SIMDJSON_HASWELL_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_HASWELL -namespace simdjson { -namespace haswell { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation( - "haswell", - "Intel/AMD AVX2", - internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2 - ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_IMPLEMENTATION_H -/* end file simdjson/haswell/implementation.h */ -namespace simdjson { -namespace internal { -static const haswell::implementation* get_haswell_singleton() { - static const haswell::implementation haswell_singleton{}; - return &haswell_singleton; -} -} // namespace internal -} // namespace simdjson -#endif - -#if SIMDJSON_IMPLEMENTATION_ICELAKE -/* including simdjson/icelake/implementation.h: #include */ -/* begin file simdjson/icelake/implementation.h */ -#ifndef SIMDJSON_ICELAKE_IMPLEMENTATION_H -#define SIMDJSON_ICELAKE_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_ICELAKE -namespace simdjson { -namespace icelake { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation( - "icelake", - "Intel/AMD AVX512", - internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2 | internal::instruction_set::AVX512F | internal::instruction_set::AVX512DQ | internal::instruction_set::AVX512CD | internal::instruction_set::AVX512BW | internal::instruction_set::AVX512VL | internal::instruction_set::AVX512VBMI2 - ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_IMPLEMENTATION_H -/* end file simdjson/icelake/implementation.h */ -namespace simdjson { -namespace internal { -static const icelake::implementation* get_icelake_singleton() { - static const icelake::implementation icelake_singleton{}; - return &icelake_singleton; -} -} // namespace internal -} // namespace simdjson -#endif - -#if SIMDJSON_IMPLEMENTATION_PPC64 -/* including simdjson/ppc64/implementation.h: #include */ -/* begin file simdjson/ppc64/implementation.h */ -#ifndef SIMDJSON_PPC64_IMPLEMENTATION_H -#define SIMDJSON_PPC64_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - -/** - * Implementation for ALTIVEC (PPC64). - */ -namespace ppc64 { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() - : simdjson::implementation("ppc64", "PPC64 ALTIVEC", - internal::instruction_set::ALTIVEC) {} - - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, size_t max_length, - std::unique_ptr &dst) - const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, - uint8_t *dst, - size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, - size_t len) const noexcept final; -}; - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_IMPLEMENTATION_H -/* end file simdjson/ppc64/implementation.h */ -namespace simdjson { -namespace internal { -static const ppc64::implementation* get_ppc64_singleton() { - static const ppc64::implementation ppc64_singleton{}; - return &ppc64_singleton; -} -} // namespace internal -} // namespace simdjson -#endif // SIMDJSON_IMPLEMENTATION_PPC64 - -#if SIMDJSON_IMPLEMENTATION_WESTMERE -/* including simdjson/westmere/implementation.h: #include */ -/* begin file simdjson/westmere/implementation.h */ -#ifndef SIMDJSON_WESTMERE_IMPLEMENTATION_H -#define SIMDJSON_WESTMERE_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE -namespace simdjson { -namespace westmere { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation("westmere", "Intel/AMD SSE4.2", internal::instruction_set::SSE42 | internal::instruction_set::PCLMULQDQ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_IMPLEMENTATION_H -/* end file simdjson/westmere/implementation.h */ -namespace simdjson { -namespace internal { -static const simdjson::westmere::implementation* get_westmere_singleton() { - static const simdjson::westmere::implementation westmere_singleton{}; - return &westmere_singleton; -} -} // namespace internal -} // namespace simdjson -#endif // SIMDJSON_IMPLEMENTATION_WESTMERE - -#if SIMDJSON_IMPLEMENTATION_LASX -/* including simdjson/lasx/implementation.h: #include */ -/* begin file simdjson/lasx/implementation.h */ -#ifndef SIMDJSON_LASX_IMPLEMENTATION_H -#define SIMDJSON_LASX_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation("lasx", "LoongArch ASX", internal::instruction_set::LASX) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_LASX_IMPLEMENTATION_H -/* end file simdjson/lasx/implementation.h */ -namespace simdjson { -namespace internal { -static const simdjson::lasx::implementation* get_lasx_singleton() { - static const simdjson::lasx::implementation lasx_singleton{}; - return &lasx_singleton; -} -} // namespace internal -} // namespace simdjson -#endif // SIMDJSON_IMPLEMENTATION_LASX - -#if SIMDJSON_IMPLEMENTATION_LSX -/* including simdjson/lsx/implementation.h: #include */ -/* begin file simdjson/lsx/implementation.h */ -#ifndef SIMDJSON_LSX_IMPLEMENTATION_H -#define SIMDJSON_LSX_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation("lsx", "LoongArch SX", internal::instruction_set::LSX) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_LSX_IMPLEMENTATION_H -/* end file simdjson/lsx/implementation.h */ -namespace simdjson { -namespace internal { -static const simdjson::lsx::implementation* get_lsx_singleton() { - static const simdjson::lsx::implementation lsx_singleton{}; - return &lsx_singleton; -} -} // namespace internal -} // namespace simdjson -#endif // SIMDJSON_IMPLEMENTATION_LSX - -#if SIMDJSON_IMPLEMENTATION_RVV_VLS -/* including simdjson/rvv-vls/implementation.h: #include */ -/* begin file simdjson/rvv-vls/implementation.h */ -#ifndef SIMDJSON_RVV_VLS_IMPLEMENTATION_H -#define SIMDJSON_RVV_VLS_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation( - "rvv_vls", - "RISC-V V extension", - internal::instruction_set::RVV_VLS - ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_IMPLEMENTATION_H -/* end file simdjson/rvv-vls/implementation.h */ -namespace simdjson { -namespace internal { -static const simdjson::rvv_vls::implementation* get_rvv_vls_singleton() { - static const simdjson::rvv_vls::implementation rvv_vls_singleton{}; - return &rvv_vls_singleton; -} -} // namespace internal -} // namespace simdjson -#endif // SIMDJSON_IMPLEMENTATION_RVV_VLS - -/* undefining SIMDJSON_CONDITIONAL_INCLUDE */ -#undef SIMDJSON_CONDITIONAL_INCLUDE - -namespace simdjson { -namespace internal { - -// When there is a single implementation, we should not pay a price -// for dispatching to the best implementation. We should just use the -// one we have. This is a compile-time check. -#define SIMDJSON_SINGLE_IMPLEMENTATION (SIMDJSON_IMPLEMENTATION_ICELAKE \ - + SIMDJSON_IMPLEMENTATION_HASWELL + SIMDJSON_IMPLEMENTATION_WESTMERE \ - + SIMDJSON_IMPLEMENTATION_ARM64 + SIMDJSON_IMPLEMENTATION_PPC64 \ - + SIMDJSON_IMPLEMENTATION_LSX + SIMDJSON_IMPLEMENTATION_LASX \ - + SIMDJSON_IMPLEMENTATION_RVV_VLS + SIMDJSON_IMPLEMENTATION_FALLBACK == 1) - -#if SIMDJSON_SINGLE_IMPLEMENTATION - simdjson_really_inline static const implementation* get_single_implementation() { - return -#if SIMDJSON_IMPLEMENTATION_ICELAKE - get_icelake_singleton(); -#endif -#if SIMDJSON_IMPLEMENTATION_HASWELL - get_haswell_singleton(); -#endif -#if SIMDJSON_IMPLEMENTATION_WESTMERE - get_westmere_singleton(); -#endif -#if SIMDJSON_IMPLEMENTATION_ARM64 - get_arm64_singleton(); -#endif -#if SIMDJSON_IMPLEMENTATION_PPC64 - get_ppc64_singleton(); -#endif -#if SIMDJSON_IMPLEMENTATION_LSX - get_lsx_singleton(); -#endif -#if SIMDJSON_IMPLEMENTATION_LASX - get_lasx_singleton(); -#endif -#if SIMDJSON_IMPLEMENTATION_RVV_VLS - get_rvv_vls_singleton(); -#endif -#if SIMDJSON_IMPLEMENTATION_FALLBACK - get_fallback_singleton(); -#endif -} -#endif - -// Static array of known implementations. We're hoping these get baked into the executable -// without requiring a static initializer. - -/** - * @private Detects best supported implementation on first use, and sets it - */ -class detect_best_supported_implementation_on_first_use final : public implementation { -public: - std::string name() const noexcept final { return set_best()->name(); } - std::string description() const noexcept final { return set_best()->description(); } - uint32_t required_instruction_sets() const noexcept final { return set_best()->required_instruction_sets(); } - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final { - return set_best()->create_dom_parser_implementation(capacity, max_length, dst); - } - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final { - return set_best()->minify(buf, len, dst, dst_len); - } - simdjson_warn_unused bool validate_utf8(const char * buf, size_t len) const noexcept final override { - return set_best()->validate_utf8(buf, len); - } - simdjson_inline detect_best_supported_implementation_on_first_use() noexcept : implementation("best_supported_detector", "Detects the best supported implementation and sets it", 0) {} -private: - const implementation *set_best() const noexcept; -}; - -static_assert(std::is_trivially_destructible::value, "detect_best_supported_implementation_on_first_use should be trivially destructible"); - -static const std::initializer_list& get_available_implementation_pointers() { - static const std::initializer_list available_implementation_pointers { -#if SIMDJSON_IMPLEMENTATION_ICELAKE - get_icelake_singleton(), -#endif -#if SIMDJSON_IMPLEMENTATION_HASWELL - get_haswell_singleton(), -#endif -#if SIMDJSON_IMPLEMENTATION_WESTMERE - get_westmere_singleton(), -#endif -#if SIMDJSON_IMPLEMENTATION_ARM64 - get_arm64_singleton(), -#endif -#if SIMDJSON_IMPLEMENTATION_PPC64 - get_ppc64_singleton(), -#endif -#if SIMDJSON_IMPLEMENTATION_LASX - get_lasx_singleton(), -#endif -#if SIMDJSON_IMPLEMENTATION_LSX - get_lsx_singleton(), -#endif -#if SIMDJSON_IMPLEMENTATION_RVV_VLS - get_rvv_vls_singleton(), -#endif -#if SIMDJSON_IMPLEMENTATION_FALLBACK - get_fallback_singleton(), -#endif - }; // available_implementation_pointers - return available_implementation_pointers; -} - -// So we can return UNSUPPORTED_ARCHITECTURE from the parser when there is no support -class unsupported_implementation final : public implementation { -public: - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t, - size_t, - std::unique_ptr& - ) const noexcept final { - return UNSUPPORTED_ARCHITECTURE; - } - simdjson_warn_unused error_code minify(const uint8_t *, size_t, uint8_t *, size_t &) const noexcept final override { - return UNSUPPORTED_ARCHITECTURE; - } - simdjson_warn_unused bool validate_utf8(const char *, size_t) const noexcept final override { - return false; // Just refuse to validate. Given that we have a fallback implementation - // it seems unlikely that unsupported_implementation will ever be used. If it is used, - // then it will flag all strings as invalid. The alternative is to return an error_code - // from which the user has to figure out whether the string is valid UTF-8... which seems - // like a lot of work just to handle the very unlikely case that we have an unsupported - // implementation. And, when it does happen (that we have an unsupported implementation), - // what are the chances that the programmer has a fallback? Given that *we* provide the - // fallback, it implies that the programmer would need a fallback for our fallback. - } - unsupported_implementation() : implementation("unsupported", "Unsupported CPU (no detected SIMD instructions)", 0) {} -}; - -static_assert(std::is_trivially_destructible::value, "unsupported_singleton should be trivially destructible"); - -const unsupported_implementation* get_unsupported_singleton() { - static const unsupported_implementation unsupported_singleton{}; - return &unsupported_singleton; -} - -size_t available_implementation_list::size() const noexcept { - return internal::get_available_implementation_pointers().size(); -} -const implementation * const *available_implementation_list::begin() const noexcept { - return internal::get_available_implementation_pointers().begin(); -} -const implementation * const *available_implementation_list::end() const noexcept { - return internal::get_available_implementation_pointers().end(); -} -const implementation *available_implementation_list::detect_best_supported() const noexcept { - // They are prelisted in priority order, so we just go down the list - uint32_t supported_instruction_sets = internal::detect_supported_architectures(); - for (const implementation *impl : internal::get_available_implementation_pointers()) { - uint32_t required_instruction_sets = impl->required_instruction_sets(); - if ((supported_instruction_sets & required_instruction_sets) == required_instruction_sets) { return impl; } - } - return get_unsupported_singleton(); // this should never happen? -} - -const implementation *detect_best_supported_implementation_on_first_use::set_best() const noexcept { - SIMDJSON_PUSH_DISABLE_WARNINGS - SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe - char *force_implementation_name = getenv("SIMDJSON_FORCE_IMPLEMENTATION"); - SIMDJSON_POP_DISABLE_WARNINGS - - if (force_implementation_name) { - auto force_implementation = get_available_implementations()[force_implementation_name]; - if (force_implementation) { - return get_active_implementation() = force_implementation; - } else { - // Note: abort() and stderr usage within the library is forbidden. - return get_active_implementation() = get_unsupported_singleton(); - } - } - return get_active_implementation() = get_available_implementations().detect_best_supported(); -} - -} // namespace internal - -SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list& get_available_implementations() { - static const internal::available_implementation_list available_implementations{}; - return available_implementations; -} - -SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr& get_active_implementation() { -#if SIMDJSON_SINGLE_IMPLEMENTATION - // We immediately select the only implementation we have, skipping the - // detect_best_supported_implementation_on_first_use_singleton. - static internal::atomic_ptr active_implementation{internal::get_single_implementation()}; - return active_implementation; -#else - static const internal::detect_best_supported_implementation_on_first_use detect_best_supported_implementation_on_first_use_singleton; - static internal::atomic_ptr active_implementation{&detect_best_supported_implementation_on_first_use_singleton}; - return active_implementation; -#endif -} - -simdjson_warn_unused error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept { - return get_active_implementation()->minify(reinterpret_cast(buf), len, reinterpret_cast(dst), dst_len); -} -simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept { - return get_active_implementation()->validate_utf8(buf, len); -} -const implementation * builtin_implementation() { - static const implementation * builtin_impl = get_available_implementations()[SIMDJSON_STRINGIFY(SIMDJSON_BUILTIN_IMPLEMENTATION)]; - assert(builtin_impl); - return builtin_impl; -} - -} // namespace simdjson - -#endif // SIMDJSON_SRC_IMPLEMENTATION_CPP -/* end file implementation.cpp */ - -/* defining SIMDJSON_CONDITIONAL_INCLUDE */ -#define SIMDJSON_CONDITIONAL_INCLUDE - -#if SIMDJSON_IMPLEMENTATION_ARM64 -/* including arm64.cpp: #include */ -/* begin file arm64.cpp */ -#ifndef SIMDJSON_SRC_ARM64_CPP -#define SIMDJSON_SRC_ARM64_CPP - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* including simdjson/arm64.h: #include */ -/* begin file simdjson/arm64.h */ -#ifndef SIMDJSON_ARM64_H -#define SIMDJSON_ARM64_H - -/* including simdjson/arm64/begin.h: #include "simdjson/arm64/begin.h" */ -/* begin file simdjson/arm64/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "arm64" */ -#define SIMDJSON_IMPLEMENTATION arm64 -/* including simdjson/arm64/base.h: #include "simdjson/arm64/base.h" */ -/* begin file simdjson/arm64/base.h */ -#ifndef SIMDJSON_ARM64_BASE_H -#define SIMDJSON_ARM64_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * Implementation for NEON (ARMv8). - */ -namespace arm64 { - -class implementation; - -namespace { -namespace simd { -template struct simd8; -template struct simd8x64; -} // namespace simd -} // unnamed namespace - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_ARM64_BASE_H -/* end file simdjson/arm64/base.h */ -/* including simdjson/arm64/intrinsics.h: #include "simdjson/arm64/intrinsics.h" */ -/* begin file simdjson/arm64/intrinsics.h */ -#ifndef SIMDJSON_ARM64_INTRINSICS_H -#define SIMDJSON_ARM64_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This should be the correct header whether -// you use visual studio or other compilers. -#include - -static_assert(sizeof(uint8x16_t) <= simdjson::SIMDJSON_PADDING, "insufficient padding for arm64"); - -#endif // SIMDJSON_ARM64_INTRINSICS_H -/* end file simdjson/arm64/intrinsics.h */ -/* including simdjson/arm64/bitmanipulation.h: #include "simdjson/arm64/bitmanipulation.h" */ -/* begin file simdjson/arm64/bitmanipulation.h */ -#ifndef SIMDJSON_ARM64_BITMANIPULATION_H -#define SIMDJSON_ARM64_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -// We sometimes call leading_zeroes on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -// Applies only when SIMDJSON_PREFER_REVERSE_BITS is defined and true. -// (See below.) -SIMDJSON_NO_SANITIZE_UNDEFINED -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int count_ones(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); -#else - // if the system supports SVE or CSSC, __builtin_popcountll - // might be compiled to fewer single instructions. For CSSC, - // __builtin_popcountll is compiled to a single instruction. - return __builtin_popcountll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO -} - - -#if defined(__GNUC__) // catches clang and gcc -/** - * ARM has a fast 64-bit "bit reversal function" that is handy. However, - * it is not generally available as an intrinsic function under Visual - * Studio (though this might be changing). Even under clang/gcc, we - * apparently need to invoke inline assembly. - */ -/* - * We use SIMDJSON_PREFER_REVERSE_BITS as a hint that algorithms that - * work well with bit reversal may use it. - */ -#define SIMDJSON_PREFER_REVERSE_BITS 1 - -/* reverse the bits */ -simdjson_inline uint64_t reverse_bits(uint64_t input_num) { - uint64_t rev_bits; - __asm("rbit %0, %1" : "=r"(rev_bits) : "r"(input_num)); - return rev_bits; -} - -/** - * Flips bit at index 63 - lz. Thus if you have 'leading_zeroes' leading zeroes, - * then this will set to zero the leading bit. It is possible for leading_zeroes to be - * greating or equal to 63 in which case we trigger undefined behavior, but the output - * of such undefined behavior is never used. - **/ -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) { - return rev_bits ^ (uint64_t(0x8000000000000000) >> leading_zeroes); -} - -#endif - - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_ARM64_BITMANIPULATION_H -/* end file simdjson/arm64/bitmanipulation.h */ -/* including simdjson/arm64/bitmask.h: #include "simdjson/arm64/bitmask.h" */ -/* begin file simdjson/arm64/bitmask.h */ -#ifndef SIMDJSON_ARM64_BITMASK_H -#define SIMDJSON_ARM64_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { - ///////////// - // We could do this with PMULL, but it is apparently slow. - // - //#ifdef __ARM_FEATURE_CRYPTO // some ARM processors lack this extension - //return vmull_p64(-1ULL, bitmask); - //#else - // Analysis by @sebpop: - // When diffing the assembly for src/stage1_find_marks.cpp I see that the eors are all spread out - // in between other vector code, so effectively the extra cycles of the sequence do not matter - // because the GPR units are idle otherwise and the critical path is on the FP side. - // Also the PMULL requires two extra fmovs: GPR->FP (3 cycles in N1, 5 cycles in A72 ) - // and FP->GPR (2 cycles on N1 and 5 cycles on A72.) - /////////// - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; - return bitmask; -} - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif -/* end file simdjson/arm64/bitmask.h */ -/* including simdjson/arm64/numberparsing_defs.h: #include "simdjson/arm64/numberparsing_defs.h" */ -/* begin file simdjson/arm64/numberparsing_defs.h */ -#ifndef SIMDJSON_ARM64_NUMBERPARSING_DEFS_H -#define SIMDJSON_ARM64_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#if SIMDJSON_REGULAR_VISUAL_STUDIO && SIMDJSON_IS_ARM64 -// __umulh requires intrin.h -#include -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO && SIMDJSON_IS_ARM64 - -namespace simdjson { -namespace arm64 { -namespace numberparsing { - -// we don't have SSE, so let us use a scalar function -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - uint64_t val; - std::memcpy(&val, chars, sizeof(uint64_t)); - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace arm64 -} // namespace simdjson - -#ifndef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_IS_BIG_ENDIAN -#define SIMDJSON_SWAR_NUMBER_PARSING 0 -#else -#define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif -#endif - -#endif // SIMDJSON_ARM64_NUMBERPARSING_DEFS_H -/* end file simdjson/arm64/numberparsing_defs.h */ -/* including simdjson/arm64/simd.h: #include "simdjson/arm64/simd.h" */ -/* begin file simdjson/arm64/simd.h */ -#ifndef SIMDJSON_ARM64_SIMD_H -#define SIMDJSON_ARM64_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace simd { - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -namespace { -// Start of private section with Visual Studio workaround - - -#ifndef simdjson_make_uint8x16_t -#define simdjson_make_uint8x16_t(x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, \ - x13, x14, x15, x16) \ - ([=]() { \ - uint8_t array[16] = {x1, x2, x3, x4, x5, x6, x7, x8, \ - x9, x10, x11, x12, x13, x14, x15, x16}; \ - return vld1q_u8(array); \ - }()) -#endif -#ifndef simdjson_make_int8x16_t -#define simdjson_make_int8x16_t(x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, \ - x13, x14, x15, x16) \ - ([=]() { \ - int8_t array[16] = {x1, x2, x3, x4, x5, x6, x7, x8, \ - x9, x10, x11, x12, x13, x14, x15, x16}; \ - return vld1q_s8(array); \ - }()) -#endif - -#ifndef simdjson_make_uint8x8_t -#define simdjson_make_uint8x8_t(x1, x2, x3, x4, x5, x6, x7, x8) \ - ([=]() { \ - uint8_t array[8] = {x1, x2, x3, x4, x5, x6, x7, x8}; \ - return vld1_u8(array); \ - }()) -#endif -#ifndef simdjson_make_int8x8_t -#define simdjson_make_int8x8_t(x1, x2, x3, x4, x5, x6, x7, x8) \ - ([=]() { \ - int8_t array[8] = {x1, x2, x3, x4, x5, x6, x7, x8}; \ - return vld1_s8(array); \ - }()) -#endif -#ifndef simdjson_make_uint16x8_t -#define simdjson_make_uint16x8_t(x1, x2, x3, x4, x5, x6, x7, x8) \ - ([=]() { \ - uint16_t array[8] = {x1, x2, x3, x4, x5, x6, x7, x8}; \ - return vld1q_u16(array); \ - }()) -#endif -#ifndef simdjson_make_int16x8_t -#define simdjson_make_int16x8_t(x1, x2, x3, x4, x5, x6, x7, x8) \ - ([=]() { \ - int16_t array[8] = {x1, x2, x3, x4, x5, x6, x7, x8}; \ - return vld1q_s16(array); \ - }()) -#endif - -// End of private section with Visual Studio workaround -} // namespace -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO - - - template - struct simd8; - - // - // Base class of simd8 and simd8, both of which use uint8x16_t internally. - // - template> - struct base_u8 { - uint8x16_t value; - static const int SIZE = sizeof(value); - - // Conversion from/to SIMD register - simdjson_inline base_u8(const uint8x16_t _value) : value(_value) {} - simdjson_inline operator const uint8x16_t&() const { return this->value; } - simdjson_inline operator uint8x16_t&() { return this->value; } - - // Bit operations - simdjson_inline simd8 operator|(const simd8 other) const { return vorrq_u8(*this, other); } - simdjson_inline simd8 operator&(const simd8 other) const { return vandq_u8(*this, other); } - simdjson_inline simd8 operator^(const simd8 other) const { return veorq_u8(*this, other); } - simdjson_inline simd8 bit_andnot(const simd8 other) const { return vbicq_u8(*this, other); } - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - simdjson_inline simd8& operator|=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline simd8& operator&=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline simd8& operator^=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast ^ other; return *this_cast; } - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return vceqq_u8(lhs, rhs); } - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return vextq_u8(prev_chunk, *this, 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base_u8 { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - static simdjson_inline simd8 splat(bool _value) { return vmovq_n_u8(uint8_t(-(!!_value))); } - - simdjson_inline simd8(const uint8x16_t _value) : base_u8(_value) {} - // False constructor - simdjson_inline simd8() : simd8(vdupq_n_u8(0)) {} - // Splat constructor - simdjson_inline simd8(bool _value) : simd8(splat(_value)) {} - - // We return uint32_t instead of uint16_t because that seems to be more efficient for most - // purposes (cutting it down to uint16_t costs performance in some compilers). - simdjson_inline uint32_t to_bitmask() const { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - const uint8x16_t bit_mask = simdjson_make_uint8x16_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80); -#else - const uint8x16_t bit_mask = {0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80}; -#endif - auto minput = *this & bit_mask; - uint8x16_t tmp = vpaddq_u8(minput, minput); - tmp = vpaddq_u8(tmp, tmp); - tmp = vpaddq_u8(tmp, tmp); - return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); - } - // Returns 4-bit out of each byte, alternating between the high 4 bits and low - // bits result it is 64 bit. - simdjson_inline uint64_t to_bitmask64() const { - return vget_lane_u64( - vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); - } - // Integer umaxv, not a float compare: flush-to-zero would treat a denormal as zero. - simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } - }; - - // Unsigned bytes - template<> - struct simd8: base_u8 { - static simdjson_inline uint8x16_t splat(uint8_t _value) { return vmovq_n_u8(_value); } - static simdjson_inline uint8x16_t zero() { return vdupq_n_u8(0); } - static simdjson_inline uint8x16_t load(const uint8_t* values) { return vld1q_u8(values); } - - simdjson_inline simd8(const uint8x16_t _value) : base_u8(_value) {} - // Zero constructor - simdjson_inline simd8() : simd8(zero()) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Member-by-member initialization -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(simdjson_make_uint8x16_t( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} -#else - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(uint8x16_t{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} -#endif - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Store to array - simdjson_inline void store(uint8_t dst[16]) const { return vst1q_u8(dst, *this); } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return vqaddq_u8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return vqsubq_u8(*this, other); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return vaddq_u8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return vsubq_u8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } - - // Order-specific operations - simdjson_inline uint8_t max_val() const { return vmaxvq_u8(*this); } - simdjson_inline uint8_t min_val() const { return vminvq_u8(*this); } - simdjson_inline simd8 max_val(const simd8 other) const { return vmaxq_u8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return vminq_u8(*this, other); } - simdjson_inline simd8 operator<=(const simd8 other) const { return vcleq_u8(*this, other); } - simdjson_inline simd8 operator>=(const simd8 other) const { return vcgeq_u8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return vcltq_u8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return vcgtq_u8(*this, other); } - // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. For ARM, returns all 1's. - simdjson_inline simd8 gt_bits(const simd8 other) const { return simd8(*this > other); } - // Same as <, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. For ARM, returns all 1's. - simdjson_inline simd8 lt_bits(const simd8 other) const { return simd8(*this < other); } - - // Bit-specific operations - simdjson_inline simd8 any_bits_set(simd8 bits) const { return vtstq_u8(*this, bits); } - simdjson_inline bool any_bits_set_anywhere() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } - template - simdjson_inline simd8 shr() const { return vshrq_n_u8(*this, N); } - template - simdjson_inline simd8 shl() const { return vshlq_n_u8(*this, N); } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return lookup_table.apply_lookup_16_to(*this); - } - - // Returns 4-bit out of each byte, alternating between the high 4 bits and low - // bits result it is 64 bit. - simdjson_inline uint64_t to_bitmask64() const { - return vget_lane_u64( - vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); - } - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint16_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - uint64x2_t shufmask64 = {thintable_epi8[mask1], thintable_epi8[mask2]}; - uint8x16_t shufmask = vreinterpretq_u8_u64(shufmask64); - // we increment by 0x08 the second half of the mask -#if SIMDJSON_REGULAR_VISUAL_STUDIO - uint8x16_t inc = simdjson_make_uint8x16_t(0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08); -#else - uint8x16_t inc = {0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08}; -#endif - shufmask = vaddq_u8(shufmask, inc); - // this is the version "nearly pruned" - uint8x16_t pruned = vqtbl1q_u8(*this, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - uint8x16_t compactmask = vld1q_u8(reinterpret_cast(pshufb_combine_table + pop1 * 8)); - uint8x16_t answer = vqtbl1q_u8(pruned, compactmask); - vst1q_u8(reinterpret_cast(output), answer); - } - - // Copies all bytes corresponding to a 0 in the low half of the mask (interpreted as a - // bitset) to output1, then those corresponding to a 0 in the high half to output2. - template - simdjson_inline void compress_halves(uint16_t mask, L *output1, L *output2) const { - using internal::thintable_epi8; - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - uint8x8_t compactmask1 = vcreate_u8(thintable_epi8[mask1]); - uint8x8_t compactmask2 = vcreate_u8(thintable_epi8[mask2]); - // we increment by 0x08 the second half of the mask -#if SIMDJSON_REGULAR_VISUAL_STUDIO - uint8x8_t inc = simdjson_make_uint8x8_t(0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08); -#else - uint8x8_t inc = {0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08}; -#endif - compactmask2 = vadd_u8(compactmask2, inc); - // store each result (with the second store possibly overlapping the first) - vst1_u8((uint8_t*)output1, vqtbl1_u8(*this, compactmask1)); - vst1_u8((uint8_t*)output2, vqtbl1_u8(*this, compactmask2)); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - - template - simdjson_inline simd8 apply_lookup_16_to(const simd8 original) { - return vqtbl1q_u8(*this, simd8(original)); - } - }; - - // Signed bytes - template<> - struct simd8 { - int8x16_t value; - - static simdjson_inline simd8 splat(int8_t _value) { return vmovq_n_s8(_value); } - static simdjson_inline simd8 zero() { return vdupq_n_s8(0); } - static simdjson_inline simd8 load(const int8_t values[16]) { return vld1q_s8(values); } - - // Conversion from/to SIMD register - simdjson_inline simd8(const int8x16_t _value) : value{_value} {} - simdjson_inline operator const int8x16_t&() const { return this->value; } - simdjson_inline operator int8x16_t&() { return this->value; } - - // Zero constructor - simdjson_inline simd8() : simd8(zero()) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} - // Member-by-member initialization -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(simdjson_make_int8x16_t( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} -#else - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(int8x16_t{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} -#endif - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Store to array - simdjson_inline void store(int8_t dst[16]) const { return vst1q_s8(dst, *this); } - - // Explicit conversion to/from unsigned - // - // Under Visual Studio/ARM64 uint8x16_t and int8x16_t are apparently the same type. - // In theory, we could check this occurrence with std::same_as and std::enabled_if but it is C++14 - // and relatively ugly and hard to read. -#ifndef SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline explicit simd8(const uint8x16_t other): simd8(vreinterpretq_s8_u8(other)) {} -#endif - simdjson_inline explicit operator simd8() const { return vreinterpretq_u8_s8(this->value); } - - // Math - simdjson_inline simd8 operator+(const simd8 other) const { return vaddq_s8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return vsubq_s8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return vmaxq_s8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return vminq_s8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return vcgtq_s8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return vcltq_s8(*this, other); } - simdjson_inline simd8 operator==(const simd8 other) const { return vceqq_s8(*this, other); } - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return vextq_s8(prev_chunk, *this, 16 - N); - } - - // Perform a lookup assuming no value is larger than 16 - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return lookup_table.apply_lookup_16_to(*this); - } - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - - template - simdjson_inline simd8 apply_lookup_16_to(const simd8 original) { - return vqtbl1q_s8(*this, simd8(original)); - } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "ARM kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - this->chunks[2].store(ptr+sizeof(simd8)*2); - this->chunks[3].store(ptr+sizeof(simd8)*3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - uint64_t popcounts = vget_lane_u64(vreinterpret_u64_u8(vcnt_u8(vcreate_u8(~mask))), 0); - // compute the prefix sum of the popcounts of each byte - uint64_t offsets = popcounts * 0x0101010101010101; - this->chunks[0].compress_halves(uint16_t(mask), output, &output[popcounts & 0xFF]); - this->chunks[1].compress_halves(uint16_t(mask >> 16), &output[(offsets >> 8) & 0xFF], &output[(offsets >> 16) & 0xFF]); - this->chunks[2].compress_halves(uint16_t(mask >> 32), &output[(offsets >> 24) & 0xFF], &output[(offsets >> 32) & 0xFF]); - this->chunks[3].compress_halves(uint16_t(mask >> 48), &output[(offsets >> 40) & 0xFF], &output[(offsets >> 48) & 0xFF]); - return offsets >> 56; - } - - simdjson_inline uint64_t to_bitmask() const { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - const uint8x16_t bit_mask = simdjson_make_uint8x16_t( - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80 - ); -#else - const uint8x16_t bit_mask = { - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80 - }; -#endif - // Add each of the elements next to each other, successively, to stuff each 8 byte mask into one. - uint8x16_t sum0 = vpaddq_u8(this->chunks[0] & bit_mask, this->chunks[1] & bit_mask); - uint8x16_t sum1 = vpaddq_u8(this->chunks[2] & bit_mask, this->chunks[3] & bit_mask); - sum0 = vpaddq_u8(sum0, sum1); - sum0 = vpaddq_u8(sum0, sum0); - return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_ARM64_SIMD_H -/* end file simdjson/arm64/simd.h */ -/* including simdjson/arm64/stringparsing_defs.h: #include "simdjson/arm64/stringparsing_defs.h" */ -/* begin file simdjson/arm64/stringparsing_defs.h */ -#ifndef SIMDJSON_ARM64_STRINGPARSING_DEFS_H -#define SIMDJSON_ARM64_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + sizeof(v0)); - v0.store(dst); - v1.store(dst + sizeof(v0)); - - // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on ARM; therefore, we - // smash them together into a 64-byte mask and get the bitmask from there. - uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; -} - - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_ARM64_STRINGPARSING_DEFS_H -/* end file simdjson/arm64/stringparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 -/* end file simdjson/arm64/begin.h */ -/* including simdjson/generic/amalgamated.h for arm64: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for arm64 */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for arm64: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for arm64 */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for arm64 */ -/* including simdjson/generic/jsoncharutils.h for arm64: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for arm64 */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for arm64 */ -/* including simdjson/generic/atomparsing.h for arm64: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for arm64 */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace arm64 { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for arm64 */ -/* including simdjson/generic/dom_parser_implementation.h for arm64: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for arm64 */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace arm64 -} // namespace simdjson - -namespace simdjson { -namespace arm64 { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for arm64 */ -/* including simdjson/generic/implementation_simdjson_result_base.h for arm64: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for arm64 */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for arm64 */ -/* including simdjson/generic/numberparsing.h for arm64: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for arm64 */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace arm64 { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for arm64 */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for arm64: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for arm64 */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for arm64 */ -/* end file simdjson/generic/amalgamated.h for arm64 */ -/* including simdjson/arm64/end.h: #include "simdjson/arm64/end.h" */ -/* begin file simdjson/arm64/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT -/* undefining SIMDJSON_IMPLEMENTATION from "arm64" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/arm64/end.h */ - -#endif // SIMDJSON_ARM64_H -/* end file simdjson/arm64.h */ -/* including simdjson/arm64/implementation.h: #include */ -/* begin file simdjson/arm64/implementation.h */ -#ifndef SIMDJSON_ARM64_IMPLEMENTATION_H -#define SIMDJSON_ARM64_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation("arm64", "ARM NEON", internal::instruction_set::NEON) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_ARM64_IMPLEMENTATION_H -/* end file simdjson/arm64/implementation.h */ - -// defining SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER allows us to provide our own bit_indexer::write -#define SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -/* including simdjson/arm64/begin.h: #include */ -/* begin file simdjson/arm64/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "arm64" */ -#define SIMDJSON_IMPLEMENTATION arm64 -/* including simdjson/arm64/base.h: #include "simdjson/arm64/base.h" */ -/* begin file simdjson/arm64/base.h */ -#ifndef SIMDJSON_ARM64_BASE_H -#define SIMDJSON_ARM64_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * Implementation for NEON (ARMv8). - */ -namespace arm64 { - -class implementation; - -namespace { -namespace simd { -template struct simd8; -template struct simd8x64; -} // namespace simd -} // unnamed namespace - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_ARM64_BASE_H -/* end file simdjson/arm64/base.h */ -/* including simdjson/arm64/intrinsics.h: #include "simdjson/arm64/intrinsics.h" */ -/* begin file simdjson/arm64/intrinsics.h */ -#ifndef SIMDJSON_ARM64_INTRINSICS_H -#define SIMDJSON_ARM64_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This should be the correct header whether -// you use visual studio or other compilers. -#include - -static_assert(sizeof(uint8x16_t) <= simdjson::SIMDJSON_PADDING, "insufficient padding for arm64"); - -#endif // SIMDJSON_ARM64_INTRINSICS_H -/* end file simdjson/arm64/intrinsics.h */ -/* including simdjson/arm64/bitmanipulation.h: #include "simdjson/arm64/bitmanipulation.h" */ -/* begin file simdjson/arm64/bitmanipulation.h */ -#ifndef SIMDJSON_ARM64_BITMANIPULATION_H -#define SIMDJSON_ARM64_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -// We sometimes call leading_zeroes on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -// Applies only when SIMDJSON_PREFER_REVERSE_BITS is defined and true. -// (See below.) -SIMDJSON_NO_SANITIZE_UNDEFINED -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int count_ones(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); -#else - // if the system supports SVE or CSSC, __builtin_popcountll - // might be compiled to fewer single instructions. For CSSC, - // __builtin_popcountll is compiled to a single instruction. - return __builtin_popcountll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO -} - - -#if defined(__GNUC__) // catches clang and gcc -/** - * ARM has a fast 64-bit "bit reversal function" that is handy. However, - * it is not generally available as an intrinsic function under Visual - * Studio (though this might be changing). Even under clang/gcc, we - * apparently need to invoke inline assembly. - */ -/* - * We use SIMDJSON_PREFER_REVERSE_BITS as a hint that algorithms that - * work well with bit reversal may use it. - */ -#define SIMDJSON_PREFER_REVERSE_BITS 1 - -/* reverse the bits */ -simdjson_inline uint64_t reverse_bits(uint64_t input_num) { - uint64_t rev_bits; - __asm("rbit %0, %1" : "=r"(rev_bits) : "r"(input_num)); - return rev_bits; -} - -/** - * Flips bit at index 63 - lz. Thus if you have 'leading_zeroes' leading zeroes, - * then this will set to zero the leading bit. It is possible for leading_zeroes to be - * greating or equal to 63 in which case we trigger undefined behavior, but the output - * of such undefined behavior is never used. - **/ -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) { - return rev_bits ^ (uint64_t(0x8000000000000000) >> leading_zeroes); -} - -#endif - - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_ARM64_BITMANIPULATION_H -/* end file simdjson/arm64/bitmanipulation.h */ -/* including simdjson/arm64/bitmask.h: #include "simdjson/arm64/bitmask.h" */ -/* begin file simdjson/arm64/bitmask.h */ -#ifndef SIMDJSON_ARM64_BITMASK_H -#define SIMDJSON_ARM64_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { - ///////////// - // We could do this with PMULL, but it is apparently slow. - // - //#ifdef __ARM_FEATURE_CRYPTO // some ARM processors lack this extension - //return vmull_p64(-1ULL, bitmask); - //#else - // Analysis by @sebpop: - // When diffing the assembly for src/stage1_find_marks.cpp I see that the eors are all spread out - // in between other vector code, so effectively the extra cycles of the sequence do not matter - // because the GPR units are idle otherwise and the critical path is on the FP side. - // Also the PMULL requires two extra fmovs: GPR->FP (3 cycles in N1, 5 cycles in A72 ) - // and FP->GPR (2 cycles on N1 and 5 cycles on A72.) - /////////// - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; - return bitmask; -} - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif -/* end file simdjson/arm64/bitmask.h */ -/* including simdjson/arm64/numberparsing_defs.h: #include "simdjson/arm64/numberparsing_defs.h" */ -/* begin file simdjson/arm64/numberparsing_defs.h */ -#ifndef SIMDJSON_ARM64_NUMBERPARSING_DEFS_H -#define SIMDJSON_ARM64_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#if SIMDJSON_REGULAR_VISUAL_STUDIO && SIMDJSON_IS_ARM64 -// __umulh requires intrin.h -#include -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO && SIMDJSON_IS_ARM64 - -namespace simdjson { -namespace arm64 { -namespace numberparsing { - -// we don't have SSE, so let us use a scalar function -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - uint64_t val; - std::memcpy(&val, chars, sizeof(uint64_t)); - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace arm64 -} // namespace simdjson - -#ifndef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_IS_BIG_ENDIAN -#define SIMDJSON_SWAR_NUMBER_PARSING 0 -#else -#define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif -#endif - -#endif // SIMDJSON_ARM64_NUMBERPARSING_DEFS_H -/* end file simdjson/arm64/numberparsing_defs.h */ -/* including simdjson/arm64/simd.h: #include "simdjson/arm64/simd.h" */ -/* begin file simdjson/arm64/simd.h */ -#ifndef SIMDJSON_ARM64_SIMD_H -#define SIMDJSON_ARM64_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace simd { - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -namespace { -// Start of private section with Visual Studio workaround - - -#ifndef simdjson_make_uint8x16_t -#define simdjson_make_uint8x16_t(x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, \ - x13, x14, x15, x16) \ - ([=]() { \ - uint8_t array[16] = {x1, x2, x3, x4, x5, x6, x7, x8, \ - x9, x10, x11, x12, x13, x14, x15, x16}; \ - return vld1q_u8(array); \ - }()) -#endif -#ifndef simdjson_make_int8x16_t -#define simdjson_make_int8x16_t(x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, \ - x13, x14, x15, x16) \ - ([=]() { \ - int8_t array[16] = {x1, x2, x3, x4, x5, x6, x7, x8, \ - x9, x10, x11, x12, x13, x14, x15, x16}; \ - return vld1q_s8(array); \ - }()) -#endif - -#ifndef simdjson_make_uint8x8_t -#define simdjson_make_uint8x8_t(x1, x2, x3, x4, x5, x6, x7, x8) \ - ([=]() { \ - uint8_t array[8] = {x1, x2, x3, x4, x5, x6, x7, x8}; \ - return vld1_u8(array); \ - }()) -#endif -#ifndef simdjson_make_int8x8_t -#define simdjson_make_int8x8_t(x1, x2, x3, x4, x5, x6, x7, x8) \ - ([=]() { \ - int8_t array[8] = {x1, x2, x3, x4, x5, x6, x7, x8}; \ - return vld1_s8(array); \ - }()) -#endif -#ifndef simdjson_make_uint16x8_t -#define simdjson_make_uint16x8_t(x1, x2, x3, x4, x5, x6, x7, x8) \ - ([=]() { \ - uint16_t array[8] = {x1, x2, x3, x4, x5, x6, x7, x8}; \ - return vld1q_u16(array); \ - }()) -#endif -#ifndef simdjson_make_int16x8_t -#define simdjson_make_int16x8_t(x1, x2, x3, x4, x5, x6, x7, x8) \ - ([=]() { \ - int16_t array[8] = {x1, x2, x3, x4, x5, x6, x7, x8}; \ - return vld1q_s16(array); \ - }()) -#endif - -// End of private section with Visual Studio workaround -} // namespace -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO - - - template - struct simd8; - - // - // Base class of simd8 and simd8, both of which use uint8x16_t internally. - // - template> - struct base_u8 { - uint8x16_t value; - static const int SIZE = sizeof(value); - - // Conversion from/to SIMD register - simdjson_inline base_u8(const uint8x16_t _value) : value(_value) {} - simdjson_inline operator const uint8x16_t&() const { return this->value; } - simdjson_inline operator uint8x16_t&() { return this->value; } - - // Bit operations - simdjson_inline simd8 operator|(const simd8 other) const { return vorrq_u8(*this, other); } - simdjson_inline simd8 operator&(const simd8 other) const { return vandq_u8(*this, other); } - simdjson_inline simd8 operator^(const simd8 other) const { return veorq_u8(*this, other); } - simdjson_inline simd8 bit_andnot(const simd8 other) const { return vbicq_u8(*this, other); } - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - simdjson_inline simd8& operator|=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline simd8& operator&=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline simd8& operator^=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast ^ other; return *this_cast; } - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return vceqq_u8(lhs, rhs); } - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return vextq_u8(prev_chunk, *this, 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base_u8 { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - static simdjson_inline simd8 splat(bool _value) { return vmovq_n_u8(uint8_t(-(!!_value))); } - - simdjson_inline simd8(const uint8x16_t _value) : base_u8(_value) {} - // False constructor - simdjson_inline simd8() : simd8(vdupq_n_u8(0)) {} - // Splat constructor - simdjson_inline simd8(bool _value) : simd8(splat(_value)) {} - - // We return uint32_t instead of uint16_t because that seems to be more efficient for most - // purposes (cutting it down to uint16_t costs performance in some compilers). - simdjson_inline uint32_t to_bitmask() const { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - const uint8x16_t bit_mask = simdjson_make_uint8x16_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80); -#else - const uint8x16_t bit_mask = {0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80}; -#endif - auto minput = *this & bit_mask; - uint8x16_t tmp = vpaddq_u8(minput, minput); - tmp = vpaddq_u8(tmp, tmp); - tmp = vpaddq_u8(tmp, tmp); - return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); - } - // Returns 4-bit out of each byte, alternating between the high 4 bits and low - // bits result it is 64 bit. - simdjson_inline uint64_t to_bitmask64() const { - return vget_lane_u64( - vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); - } - // Integer umaxv, not a float compare: flush-to-zero would treat a denormal as zero. - simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } - }; - - // Unsigned bytes - template<> - struct simd8: base_u8 { - static simdjson_inline uint8x16_t splat(uint8_t _value) { return vmovq_n_u8(_value); } - static simdjson_inline uint8x16_t zero() { return vdupq_n_u8(0); } - static simdjson_inline uint8x16_t load(const uint8_t* values) { return vld1q_u8(values); } - - simdjson_inline simd8(const uint8x16_t _value) : base_u8(_value) {} - // Zero constructor - simdjson_inline simd8() : simd8(zero()) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Member-by-member initialization -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(simdjson_make_uint8x16_t( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} -#else - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(uint8x16_t{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} -#endif - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Store to array - simdjson_inline void store(uint8_t dst[16]) const { return vst1q_u8(dst, *this); } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return vqaddq_u8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return vqsubq_u8(*this, other); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return vaddq_u8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return vsubq_u8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } - - // Order-specific operations - simdjson_inline uint8_t max_val() const { return vmaxvq_u8(*this); } - simdjson_inline uint8_t min_val() const { return vminvq_u8(*this); } - simdjson_inline simd8 max_val(const simd8 other) const { return vmaxq_u8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return vminq_u8(*this, other); } - simdjson_inline simd8 operator<=(const simd8 other) const { return vcleq_u8(*this, other); } - simdjson_inline simd8 operator>=(const simd8 other) const { return vcgeq_u8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return vcltq_u8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return vcgtq_u8(*this, other); } - // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. For ARM, returns all 1's. - simdjson_inline simd8 gt_bits(const simd8 other) const { return simd8(*this > other); } - // Same as <, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. For ARM, returns all 1's. - simdjson_inline simd8 lt_bits(const simd8 other) const { return simd8(*this < other); } - - // Bit-specific operations - simdjson_inline simd8 any_bits_set(simd8 bits) const { return vtstq_u8(*this, bits); } - simdjson_inline bool any_bits_set_anywhere() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } - template - simdjson_inline simd8 shr() const { return vshrq_n_u8(*this, N); } - template - simdjson_inline simd8 shl() const { return vshlq_n_u8(*this, N); } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return lookup_table.apply_lookup_16_to(*this); - } - - // Returns 4-bit out of each byte, alternating between the high 4 bits and low - // bits result it is 64 bit. - simdjson_inline uint64_t to_bitmask64() const { - return vget_lane_u64( - vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); - } - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint16_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - uint64x2_t shufmask64 = {thintable_epi8[mask1], thintable_epi8[mask2]}; - uint8x16_t shufmask = vreinterpretq_u8_u64(shufmask64); - // we increment by 0x08 the second half of the mask -#if SIMDJSON_REGULAR_VISUAL_STUDIO - uint8x16_t inc = simdjson_make_uint8x16_t(0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08); -#else - uint8x16_t inc = {0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08}; -#endif - shufmask = vaddq_u8(shufmask, inc); - // this is the version "nearly pruned" - uint8x16_t pruned = vqtbl1q_u8(*this, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - uint8x16_t compactmask = vld1q_u8(reinterpret_cast(pshufb_combine_table + pop1 * 8)); - uint8x16_t answer = vqtbl1q_u8(pruned, compactmask); - vst1q_u8(reinterpret_cast(output), answer); - } - - // Copies all bytes corresponding to a 0 in the low half of the mask (interpreted as a - // bitset) to output1, then those corresponding to a 0 in the high half to output2. - template - simdjson_inline void compress_halves(uint16_t mask, L *output1, L *output2) const { - using internal::thintable_epi8; - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - uint8x8_t compactmask1 = vcreate_u8(thintable_epi8[mask1]); - uint8x8_t compactmask2 = vcreate_u8(thintable_epi8[mask2]); - // we increment by 0x08 the second half of the mask -#if SIMDJSON_REGULAR_VISUAL_STUDIO - uint8x8_t inc = simdjson_make_uint8x8_t(0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08); -#else - uint8x8_t inc = {0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08}; -#endif - compactmask2 = vadd_u8(compactmask2, inc); - // store each result (with the second store possibly overlapping the first) - vst1_u8((uint8_t*)output1, vqtbl1_u8(*this, compactmask1)); - vst1_u8((uint8_t*)output2, vqtbl1_u8(*this, compactmask2)); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - - template - simdjson_inline simd8 apply_lookup_16_to(const simd8 original) { - return vqtbl1q_u8(*this, simd8(original)); - } - }; - - // Signed bytes - template<> - struct simd8 { - int8x16_t value; - - static simdjson_inline simd8 splat(int8_t _value) { return vmovq_n_s8(_value); } - static simdjson_inline simd8 zero() { return vdupq_n_s8(0); } - static simdjson_inline simd8 load(const int8_t values[16]) { return vld1q_s8(values); } - - // Conversion from/to SIMD register - simdjson_inline simd8(const int8x16_t _value) : value{_value} {} - simdjson_inline operator const int8x16_t&() const { return this->value; } - simdjson_inline operator int8x16_t&() { return this->value; } - - // Zero constructor - simdjson_inline simd8() : simd8(zero()) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} - // Member-by-member initialization -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(simdjson_make_int8x16_t( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} -#else - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(int8x16_t{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} -#endif - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Store to array - simdjson_inline void store(int8_t dst[16]) const { return vst1q_s8(dst, *this); } - - // Explicit conversion to/from unsigned - // - // Under Visual Studio/ARM64 uint8x16_t and int8x16_t are apparently the same type. - // In theory, we could check this occurrence with std::same_as and std::enabled_if but it is C++14 - // and relatively ugly and hard to read. -#ifndef SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline explicit simd8(const uint8x16_t other): simd8(vreinterpretq_s8_u8(other)) {} -#endif - simdjson_inline explicit operator simd8() const { return vreinterpretq_u8_s8(this->value); } - - // Math - simdjson_inline simd8 operator+(const simd8 other) const { return vaddq_s8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return vsubq_s8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return vmaxq_s8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return vminq_s8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return vcgtq_s8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return vcltq_s8(*this, other); } - simdjson_inline simd8 operator==(const simd8 other) const { return vceqq_s8(*this, other); } - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return vextq_s8(prev_chunk, *this, 16 - N); - } - - // Perform a lookup assuming no value is larger than 16 - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return lookup_table.apply_lookup_16_to(*this); - } - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - - template - simdjson_inline simd8 apply_lookup_16_to(const simd8 original) { - return vqtbl1q_s8(*this, simd8(original)); - } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "ARM kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - this->chunks[2].store(ptr+sizeof(simd8)*2); - this->chunks[3].store(ptr+sizeof(simd8)*3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - uint64_t popcounts = vget_lane_u64(vreinterpret_u64_u8(vcnt_u8(vcreate_u8(~mask))), 0); - // compute the prefix sum of the popcounts of each byte - uint64_t offsets = popcounts * 0x0101010101010101; - this->chunks[0].compress_halves(uint16_t(mask), output, &output[popcounts & 0xFF]); - this->chunks[1].compress_halves(uint16_t(mask >> 16), &output[(offsets >> 8) & 0xFF], &output[(offsets >> 16) & 0xFF]); - this->chunks[2].compress_halves(uint16_t(mask >> 32), &output[(offsets >> 24) & 0xFF], &output[(offsets >> 32) & 0xFF]); - this->chunks[3].compress_halves(uint16_t(mask >> 48), &output[(offsets >> 40) & 0xFF], &output[(offsets >> 48) & 0xFF]); - return offsets >> 56; - } - - simdjson_inline uint64_t to_bitmask() const { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - const uint8x16_t bit_mask = simdjson_make_uint8x16_t( - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80 - ); -#else - const uint8x16_t bit_mask = { - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80 - }; -#endif - // Add each of the elements next to each other, successively, to stuff each 8 byte mask into one. - uint8x16_t sum0 = vpaddq_u8(this->chunks[0] & bit_mask, this->chunks[1] & bit_mask); - uint8x16_t sum1 = vpaddq_u8(this->chunks[2] & bit_mask, this->chunks[3] & bit_mask); - sum0 = vpaddq_u8(sum0, sum1); - sum0 = vpaddq_u8(sum0, sum0); - return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_ARM64_SIMD_H -/* end file simdjson/arm64/simd.h */ -/* including simdjson/arm64/stringparsing_defs.h: #include "simdjson/arm64/stringparsing_defs.h" */ -/* begin file simdjson/arm64/stringparsing_defs.h */ -#ifndef SIMDJSON_ARM64_STRINGPARSING_DEFS_H -#define SIMDJSON_ARM64_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + sizeof(v0)); - v0.store(dst); - v1.store(dst + sizeof(v0)); - - // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on ARM; therefore, we - // smash them together into a 64-byte mask and get the bitmask from there. - uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; -} - - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_ARM64_STRINGPARSING_DEFS_H -/* end file simdjson/arm64/stringparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 -/* end file simdjson/arm64/begin.h */ -/* including generic/amalgamated.h for arm64: #include */ -/* begin file generic/amalgamated.h for arm64 */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) -#error generic/dependencies.h must be included before generic/amalgamated.h! -#endif - -/* including generic/base.h for arm64: #include */ -/* begin file generic/base.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { - -struct json_character_block; - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_BASE_H -/* end file generic/base.h for arm64 */ -/* including generic/dom_parser_implementation.h for arm64: #include */ -/* begin file generic/dom_parser_implementation.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// Interface a dom parser implementation must fulfill -namespace simdjson { -namespace arm64 { -namespace { - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); -simdjson_inline bool is_ascii(const simd8x64& input); - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file generic/dom_parser_implementation.h for arm64 */ -/* including generic/json_character_block.h for arm64: #include */ -/* begin file generic/json_character_block.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { - -struct json_character_block { - static simdjson_inline json_character_block classify(const simd::simd8x64& in); - - simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_inline uint64_t op() const noexcept { return _op; } - simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } - - uint64_t _whitespace; - uint64_t _op; -}; - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H -/* end file generic/json_character_block.h for arm64 */ -/* end file generic/amalgamated.h for arm64 */ -/* including generic/stage1/amalgamated.h for arm64: #include */ -/* begin file generic/stage1/amalgamated.h for arm64 */ -// Stuff other things depend on -/* including generic/stage1/base.h for arm64: #include */ -/* begin file generic/stage1/base.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage1 { - -class bit_indexer; -template -struct buf_block_reader; -struct json_block; -class json_minifier; -class json_scanner; -struct json_string_block; -class json_string_scanner; -class json_structural_indexer; - -} // namespace stage1 - -namespace utf8_validation { -struct utf8_checker; -} // namespace utf8_validation - -using utf8_validation::utf8_checker; - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H -/* end file generic/stage1/base.h for arm64 */ -/* including generic/stage1/buf_block_reader.h for arm64: #include */ -/* begin file generic/stage1/buf_block_reader.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage1 { - -// Walks through a buffer in block-sized increments, loading the last part with spaces -template -struct buf_block_reader { -public: - simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t *full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t *dst) const; - /** - * Like get_remainder, for input followed by SIMDJSON_PADDING readable bytes. - * - * get_remainder writes the block with a memset and a memcpy of the remaining - * length, and the vector loads that read it back cannot be forwarded from - * those narrower stores: they wait until the stores reach the cache, which - * costs tens of cycles on every document. Here the block is put together in - * vector registers, from loads that may read into the padding, and written - * with full-width stores that the loads can forward from. - */ - simdjson_inline size_t get_remainder_padded(uint8_t *dst) const; - simdjson_inline void advance(); -private: - const uint8_t *buf; - const size_t len; - const size_t lenminusstep; - size_t idx; -}; - -template -simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} - -template -simdjson_inline size_t buf_block_reader::block_index() { return idx; } - -template -simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; -} - -template -simdjson_inline const uint8_t *buf_block_reader::full_block() const { - return &buf[idx]; -} - -template -simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { - if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; -} - -// keep_mask[128 - n + i] is 0xFF for i < n and 0 otherwise. -alignas(64) static const uint8_t keep_mask[256] = { - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF -}; - -template -simdjson_inline size_t buf_block_reader::get_remainder_padded(uint8_t *dst) const { - static_assert(STEP_SIZE <= 128, "keep_mask covers blocks of up to 128 bytes"); - static_assert(sizeof(simd::simd8) <= SIMDJSON_PADDING, "a vector load at the last byte must stay in the padding"); - const size_t remaining = len - idx; - if (remaining == 0) { return 0; } - const simd::simd8 spaces = simd::simd8::splat(0x20); - for (size_t off = 0; off < STEP_SIZE; off += sizeof(simd::simd8)) { - simd::simd8 v = spaces; - if (off < remaining) { - // Starts before len, so it ends within the padding. - const simd::simd8 in = simd::simd8::load(buf + idx + off); - const simd::simd8 keep = simd::simd8::load(keep_mask + 128 - remaining + off); - v = spaces ^ ((in ^ spaces) & keep); - } - v.store(dst + off); - } - return remaining; -} - -template -simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for arm64 */ -/* including generic/stage1/json_escape_scanner.h for arm64: #include */ -/* begin file generic/stage1/json_escape_scanner.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage1 { - -/** - * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). - */ -struct json_escape_scanner { - /** The actual escape characters (the backslashes themselves). */ - uint64_t next_is_escaped = 0ULL; - - struct escaped_and_escape { - /** - * Mask of escaped characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 0100100010100101000 - * n \ \ n \ \ - * ``` - */ - uint64_t escaped; - /** - * Mask of escape characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 1001000101001010001 - * \ \ \ \ \ \ \ - * ``` - */ - uint64_t escape; - }; - - /** - * Get a mask of both escape and escaped characters (the characters following a backslash). - * - * @param potential_escape A mask of the character that can escape others (but could be - * escaped itself). e.g. block.eq('\\') - */ - simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { - -#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT - if (!backslash) { return {next_escaped_without_backslashes(), 0}; } -#endif - - // | | Mask (shows characters instead of 1's) | Depth | Instructions | - // |--------------------------------|----------------------------------------|-------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | - // | | ` even odd even odd odd` | | | - // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) - // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) - // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) - // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () - // (*) this is not needed until the next iteration - uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); - uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); - uint64_t escape = escape_and_terminal_code & backslash; - this->next_is_escaped = escape >> 63; - return {escaped, escape}; - } - -private: - static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; - - simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { - uint64_t escaped = this->next_is_escaped; - this->next_is_escaped = 0; - return escaped; - } - - /** - * Returns a mask of the next escape characters (masking out escaped backslashes), along with - * any non-backslash escape codes. - * - * \n \\n \\\n \\\\n returns: - * \n \ \ \n \ \ - * 11 100 1011 10100 - * - * You are expected to mask out the first bit yourself if the previous block had a trailing - * escape. - * - * & the result with potential_escape to get just the escape characters. - * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. - */ - static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { - // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: - // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be - // inverted (\\\ would be 010 instead of 101). - // - // ``` - // string: | ____\\\\_\\\\_____ | - // maybe_escaped | ODD | \ \ \ \ | - // even-aligned ^^^ ^^^^ odd-aligned - // ``` - // - // Taking that into account, our basic strategy is: - // - // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for - // odd-aligned runs. - // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the - // odd bits in odd-aligned runs. - // 3. & with backslash to clean up any stray bits. - // runs are set to 0, and then XORing with "odd": - // - // | | Mask (shows characters instead of 1's) | Instructions | - // |--------------------------------|----------------------------------------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | - // | | ` even odd even odd odd` | - // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) - // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) - // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) - // - - // Escaped characters are characters following an escape. - uint64_t maybe_escaped = potential_escape << 1; - - // To distinguish odd from even escape sequences, therefore, we turn on any *starting* - // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) - // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. - // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. - // - All other odd bytes are 1, and even bytes are 0. - uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; - uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; - - // Now we flip all odd bytes back with xor. This: - // - Makes odd runs of backslashes go from 0000 to 1010 - // - Makes even runs of backslashes go from 1111 to 1010 - // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) - // - Resets all other bytes to 0 - return even_series_codes_and_odd_bits ^ ODD_BITS; - } -}; - -} // namespace stage1 -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_escape_scanner.h for arm64 */ -/* including generic/stage1/json_string_scanner.h for arm64: #include */ -/* begin file generic/stage1/json_string_scanner.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage1 { - -struct json_string_block { - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : - _escaped(escaped), _quote(quote), _in_string(in_string) {} - - // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } - // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } - // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } - // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } - - // escaped characters (backslashed--does not include the hex characters after \u) - uint64_t _escaped; - // real quotes (non-escaped ones) - uint64_t _quote; - // string characters (includes start quote but not end quote) - uint64_t _in_string; -}; - -// Scans blocks for string characters, storing the state necessary to do so -class json_string_scanner { -public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); - -private: - // Scans for escape characters - json_escape_scanner escape_scanner{}; - // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). - uint64_t prev_in_string = 0ULL; -}; - -// -// Return a mask of all string characters plus end quotes. -// -// prev_escaped is overflow saying whether the next character is escaped. -// prev_in_string is overflow saying whether we're still in a string. -// -// Backslash sequences outside of quotes will be detected in stage 2. -// -simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = escape_scanner.next(backslash).escaped; - const uint64_t quote = in.eq('"') & ~escaped; - - // - // prefix_xor flips on bits inside the string (and flips off the end quote). - // - // Then we xor with prev_in_string: if we were in a string already, its effect is flipped - // (characters inside strings are outside, and characters outside strings are inside). - // - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - - // - // Check if we're still in a string at the end of the box so the next block will know - // - prev_in_string = uint64_t(static_cast(in_string) >> 63); - - // Use ^ to turn the beginning quote off, and the end quote on. - - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_string_block(escaped, quote, in_string); -} - -simdjson_really_inline error_code json_string_scanner::finish() { - if (prev_in_string) { - return UNCLOSED_STRING; - } - return SUCCESS; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_string_scanner.h for arm64 */ -/* including generic/stage1/utf8_lookup4_algorithm.h for arm64: #include */ -/* begin file generic/stage1/utf8_lookup4_algorithm.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace utf8_validation { - -using namespace simd; - - simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { -// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) -// Bit 1 = Too Long (ASCII followed by continuation) -// Bit 2 = Overlong 3-byte -// Bit 4 = Surrogate -// Bit 5 = Overlong 2-byte -// Bit 7 = Two Continuations - constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ - // 11______ 11______ - constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ - constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ - constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ - constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ - constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ - constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ - // 11110100 101_____ - // 11110101 1001____ - // 11110101 101_____ - // 1111011_ 1001____ - // 1111011_ 101_____ - // 11111___ 1001____ - // 11111___ 101_____ - constexpr const uint8_t TOO_LARGE_1000 = 1<<6; - // 11110101 1000____ - // 1111011_ 1000____ - // 11111___ 1000____ - constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ - - const simd8 byte_1_high = prev1.shr<4>().lookup_16( - // 0_______ ________ - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - // 10______ ________ - TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, - // 1100____ ________ - TOO_SHORT | OVERLONG_2, - // 1101____ ________ - TOO_SHORT, - // 1110____ ________ - TOO_SHORT | OVERLONG_3 | SURROGATE, - // 1111____ ________ - TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 - ); - constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( - // ____0000 ________ - CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, - // ____0001 ________ - CARRY | OVERLONG_2, - // ____001_ ________ - CARRY, - CARRY, - - // ____0100 ________ - CARRY | TOO_LARGE, - // ____0101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____011_ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - - // ____1___ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____1101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000 - ); - const simd8 byte_2_high = input.shr<4>().lookup_16( - // ________ 0_______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - - // ________ 1000____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, - // ________ 1001____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, - // ________ 101_____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - - // ________ 11______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT - ); - return (byte_1_high & byte_1_low & byte_2_high); - } - simdjson_inline simd8 check_multibyte_lengths(const simd8 input, - const simd8 prev_input, const simd8 sc) { - simd8 prev2 = input.prev<2>(prev_input); - simd8 prev3 = input.prev<3>(prev_input); - simd8 must23 = must_be_2_3_continuation(prev2, prev3); - simd8 must23_80 = must23 & uint8_t(0x80); - return must23_80 ^ sc; - } - - // - // Return nonzero if there are incomplete multibyte characters at the end of the block: - // e.g. if there is a 4-byte character, but it's 3 bytes from the end. - // - simdjson_inline simd8 is_incomplete(const simd8 input) { - // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): - // ... 1111____ 111_____ 11______ -#if SIMDJSON_IMPLEMENTATION_ICELAKE || (SIMDJSON_IMPLEMENTATION_RVV_VLS && __riscv_v_fixed_vlen >= 512) - static const uint8_t max_array[64] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#else - static const uint8_t max_array[32] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#endif - const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); - return input.gt_bits(max_value); - } - - struct utf8_checker { - // If this is nonzero, there has been a UTF-8 error. - simd8 error; - // The last input we received - simd8 prev_input_block; - // Whether the last input we received was incomplete (used for ASCII fast path) - simd8 prev_incomplete; - - // - // Check whether the current bytes are valid UTF-8. - // - simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { - // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes - // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) - simd8 prev1 = input.prev<1>(prev_input); - simd8 sc = check_special_cases(input, prev1); - this->error |= check_multibyte_lengths(input, prev_input, sc); - } - - // The only problem that can happen at EOF is that a multibyte character is too short - // or a byte value too large in the last bytes: check_special_cases only checks for bytes - // too large in the first of two bytes. - simdjson_inline void check_eof() { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; - } - - simdjson_inline void check_next_input(const simd8x64& input) { - if(simdjson_likely(is_ascii(input))) { - this->error |= this->prev_incomplete; - } else { - // you might think that a for-loop would work, but under Visual Studio, it is not good enough. - static_assert((simd8x64::NUM_CHUNKS == 1) - ||(simd8x64::NUM_CHUNKS == 2) - || (simd8x64::NUM_CHUNKS == 4), - "We support one, two or four chunks per 64-byte block."); - SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 1) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 2) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 4) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - this->check_utf8_bytes(input.get<2>(), input.get<1>()); - this->check_utf8_bytes(input.get<3>(), input.get<2>()); - } - this->prev_incomplete = is_incomplete(input.get::NUM_CHUNKS-1>()); - this->prev_input_block = input.get::NUM_CHUNKS-1>(); - } - } - // do not forget to call check_eof! - simdjson_warn_unused simdjson_inline error_code errors() { - return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; - } - - }; // struct utf8_checker -} // namespace utf8_validation - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H -/* end file generic/stage1/utf8_lookup4_algorithm.h for arm64 */ -/* including generic/stage1/json_scanner.h for arm64: #include */ -/* begin file generic/stage1/json_scanner.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage1 { - -/** - * A block of scanned json, with information on operators and scalars. - * - * We seek to identify pseudo-structural characters. Anything that is inside - * a string must be omitted (hence & ~_string.string_tail()). - * Otherwise, pseudo-structural characters come in two forms. - * 1. We have the structural characters ([,],{,},:, comma). The - * term 'structural character' is from the JSON RFC. - * 2. We have the 'scalar pseudo-structural characters'. - * Scalars are quotes, and any character except structural characters and white space. - * - * To identify the scalar pseudo-structural characters, we must look at what comes - * before them: it must be a space, a quote or a structural characters. - * Starting with simdjson v0.3, we identify them by - * negation: we identify everything that is followed by a non-quote scalar, - * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. - */ -struct json_block { -public: - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - - /** - * The start of structurals. - * In simdjson prior to v0.3, these were called the pseudo-structural characters. - **/ - simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } - /** All JSON whitespace (i.e. not in a string) */ - simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } - - // Helpers - - /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } - /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } - - // string and escape characters - json_string_block _string; - // whitespace, structural characters ('operators'), scalars - json_character_block _characters; - // whether the previous character was a scalar - uint64_t _follows_potential_nonquote_scalar; -private: - // Potential structurals (i.e. disregarding strings) - - /** - * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". - * They may reside inside a string. - **/ - simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } - /** - * The start of non-operator runs, like 123, true and "abc". - * It main reside inside a string. - **/ - simdjson_inline uint64_t potential_scalar_start() const noexcept { - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space - // then we know that it is irrelevant structurally. - return _characters.scalar() & ~follows_potential_scalar(); - } - /** - * Whether the given character is immediately after a non-operator like 123, true. - * The characters following a quote are not included. - */ - simdjson_inline uint64_t follows_potential_scalar() const noexcept { - // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character - // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a - // white space. - // It is understood that within quoted region, anything at all could be marked (irrelevant). - return _follows_potential_nonquote_scalar; - } -}; - -/** - * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. - * - * The scanner starts by calculating two distinct things: - * - string characters (taking \" into account) - * - structural characters or 'operators' ([]{},:, comma) - * and scalars (runs of non-operators like 123, true and "abc") - * - * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: - * in particular, the operator/scalar bit will find plenty of things that are actually part of - * strings. When we're done, json_block will fuse the two together by masking out tokens that are - * part of a string. - */ -class json_scanner { -public: - json_scanner() = default; - simdjson_inline json_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_warn_unused simdjson_inline error_code finish(); - -private: - // Whether the last character of the previous iteration is part of a scalar token - // (anything except whitespace or a structural character/'operator'). - uint64_t prev_scalar = 0ULL; - json_string_scanner string_scanner{}; -}; - - -// -// Check if the current character immediately follows a matching character. -// -// For example, this checks for quotes with backslashes in front of them: -// -// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); -// -simdjson_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; -} - -simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { - json_string_block strings = string_scanner.next(in); - // identifies the white-space and the structural characters - json_character_block characters = json_character_block::classify(in); - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). - // - // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) - // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential - // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we - // may need to add an extra check when parsing strings. - // - // Performance: there are many ways to skin this cat. - const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); - uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_block( - strings,// strings is a function-local object so either it moves or the copy is elided. - characters, - follows_nonquote_scalar - ); -} - -simdjson_warn_unused simdjson_inline error_code json_scanner::finish() { - return string_scanner.finish(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H -/* end file generic/stage1/json_scanner.h for arm64 */ - -// All other declarations -/* including generic/stage1/find_next_document_index.h for arm64: #include */ -/* begin file generic/stage1/find_next_document_index.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage1 { - -/** - * This algorithm is used to quickly identify the last structural position that - * makes up a complete document. - * - * It does this by going backwards and finding the last *document boundary* (a - * place where one value follows another without a comma between them). If the - * last document (the characters after the boundary) has an equal number of - * start and end brackets, it is considered complete. - * - * Simply put, we iterate over the structural characters, starting from - * the end. We consider that we found the end of a JSON document when the - * first element of the pair is NOT one of these characters: '{' '[' ':' ',' - * and when the second element is NOT one of these characters: '}' ']' ':' ','. - * - * This simple comparison works most of the time, but it does not cover cases - * where the batch's structural indexes contain a perfect amount of documents. - * In such a case, we do not have access to the structural index which follows - * the last document, therefore, we do not have access to the second element in - * the pair, and that means we cannot identify the last document. To fix this - * issue, we keep a count of the open and closed curly/square braces we found - * while searching for the pair. When we find a pair AND the count of open and - * closed curly/square braces is the same, we know that we just passed a - * complete document, therefore the last json buffer location is the end of the - * batch. - */ -simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { - const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - const uint8_t e = parser.buf[len - 1]; - return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && - e != ' ' && e != '\t' && e != '\n' && e != '\r'; -} - -/** - * The structural index past the last one of a partial batch is the end of this - * window, not a token of it: the byte there is the first byte of the next - * window. A finished value must not be rejected because that byte is ',', ':', - * ']' or '}'. A last document that itself ends with '{', '[', ',' or ':' is - * invalid JSON, yet its brackets may balance: walking it, the On-Demand API - * could read the next batch as a value. We leave such a document to the next - * batch, so that the batch ends between two documents. - */ -simdjson_inline bool ends_with_unfinished_document(dom_parser_implementation &parser) { - const uint8_t last = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - return last == '{' || last == '[' || last == ',' || last == ':'; -} - -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { - // Variant: do not count separately, just figure out depth - if(parser.n_structural_indexes == 0) { return 0; } - auto arr_cnt = 0; - auto obj_cnt = 0; - for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { - auto idxb = parser.structural_indexes[i]; - switch (parser.buf[idxb]) { - case ':': - case ',': - continue; - case '}': - obj_cnt--; - continue; - case ']': - arr_cnt--; - continue; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - auto idxa = parser.structural_indexes[i - 1]; - switch (parser.buf[idxa]) { - case '{': - case '[': - case ':': - case ',': - continue; - } - // Last document is complete, so the next document will appear after! - if (!arr_cnt && !obj_cnt) { - return defer_last ? i : parser.n_structural_indexes; - } - // Last document is incomplete; mark the document at i + 1 as the next one - return i; - } - // If we made it to the end, we want to finish counting to see if we have a full document. - switch (parser.buf[parser.structural_indexes[0]]) { - case '}': - obj_cnt--; - break; - case ']': - arr_cnt--; - break; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - if (!arr_cnt && !obj_cnt) { - // We have a complete document. - return defer_last ? 0 : parser.n_structural_indexes; - } - return 0; -} - -/** - * Sentinel value returned to indicate a document started but didn't fit - * (CAPACITY error), as opposed to 0 which means no document content found - * (EMPTY). - */ -constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; - -/** - * For RFC 7464 JSON text sequences, filter RS from structural indexes and - * find batch boundaries. - * - * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. - * RS bytes appear in structural_indexes as they are classified as scalars. - * This function: - * 1. Scans structural_indexes to find and count RS positions - * 2. Filters RS out of structural_indexes in-place - * 3. Determines batch boundaries based on RS positions - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @param scan_len Offset past which no value start may be derived. Equals len - * unless stage 1 dropped a trailing unclosed string, whose bytes it - * never validated. - * @return The number of structural indexes to keep (after RS filtering), - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t find_next_document_index_json_sequence( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start, - size_t scan_len) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Phase 1: Scan structural_indexes to find RS positions and handle them. - // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS - // is already in structural_indexes (it's an operator). For scalars like numbers, - // the digit following RS is NOT in structural_indexes because the scanner sees - // RS as a scalar, making the digit a scalar continuation, not a start. - // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the - // actual value start position. - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_rs_pos = 0; - uint32_t rs_count = 0; - - for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { - const uint32_t pos = parser.structural_indexes[read_idx]; - if (parser.buf[pos] == 0x1E) { - // This is an RS character - find the actual JSON value start. - last_rs_pos = pos; - rs_count++; - // Skip past this RS and any whitespace *and any additional RSes* - // to locate the real value. Consecutive RSes are degenerate - // "empty records" per RFC 7464; we collapse them here. They do - // not always appear as separate entries in structural_indexes - // because the scanner groups runs of adjacent non-whitespace - // scalar bytes (including RS) into a single scalar start. - uint32_t value_start = pos + 1; - while (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { - value_start++; - } else if (c == 0x1E) { - // Collapsed empty record. Still count it so rs_count reflects - // the true number of record markers and last_rs_pos tracks - // the final one. - last_rs_pos = value_start; - rs_count++; - value_start++; - } else { - break; - } - } - // If the scanner emitted additional structurals inside the - // whitespace+RS run we just walked over (i.e., isolated RSes - // separated by whitespace), skip past them so we do not - // double-count or double-emit. - while (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] < value_start) { - read_idx++; - } - // Check if the value start is an operator (always present in - // scanner structural_indexes) or a scalar-like start (which may - // be missing from structural_indexes and must be added here). - // Note: '"' is NOT always in structural_indexes. The scanner - // classifies '"' as a scalar character and emits it as a - // structural only when it is a *scalar start* (preceded by - // whitespace or an operator). When '"' immediately follows an - // RS (which the scanner also classifies as scalar), it is - // treated as a scalar continuation and not emitted - so we - // must add it here just like any other scalar value. - if (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - const bool is_operator = - (c == '{' || c == '}' || c == '[' || c == ']' || - c == ':' || c == ','); - // If the next scanner structural is exactly at value_start, - // the scanner already emitted it (it followed whitespace) and - // we must not add a duplicate - a subsequent iteration will - // copy it into write_idx. - const bool already_emitted = - (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] == value_start); - if (!is_operator && !already_emitted) { - // Scalar value (number/true/false/null/string) - add its - // position since scanner missed it. - parser.structural_indexes[write_idx++] = value_start; - } - } - } else { - // Not RS, copy to output - parser.structural_indexes[write_idx++] = pos; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { - // Only RS markers here: the last one opens a record continuing past the - // window, so that is where the next batch resumes. - if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } - return 0; - } - if (rs_count == 0) { - // No RS found; for final batch, try generic boundary detection - return is_final ? find_next_document_index(parser) : 0; - } - - // Phase 2: Determine batch boundaries based on RS positions - - if (is_final) { - // Final batch: all documents are complete (last one ends at EOF). - // In json_sequence mode, RS markers define document boundaries, so all - // remaining structurals form complete documents. Return them all directly. - // (Calling find_next_document_index() would fail for scalar documents.) - return parser.n_structural_indexes; - } - - // Partial batch: need to find complete documents only. - // A document starting at an RS is complete if there is another RS after it. - next_batch_start = last_rs_pos; - - if (rs_count < 2) { - // Only one RS, so we have at most one document that may be incomplete. - // We cannot confirm it is complete without another RS. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least 2 RS markers. The last complete document ends before last_rs_pos. - - // Find the structural index cutoff: keep only structurals < last_rs_pos. - // Since we already filtered RS, all remaining structurals are valid. - // We iterate backward to find the last structural before last_rs_pos. - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_rs_pos) { - keep_count = i; - break; - } - } - - // No structurals before the last RS - no complete documents - if (keep_count == 0) { return 0; } - - // All documents before the last RS are complete by definition (the next RS - // confirms their end). No need to call find_next_document_index() which - // would fail for scalar documents like `1` or `"hello"`. - return keep_count; -} - -/** - * Filter comma-delimited documents by removing root-level commas from - * structural indexes. - * - * For comma-delimited format like `{...},{...},{...}`, we need to remove - * the commas that separate documents (depth 0) while preserving commas - * inside arrays and objects (depth > 0). - * - * After filtering, the structural indexes look like whitespace-delimited - * documents, so find_next_document_index() works unchanged. - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @return The number of structural indexes to keep, - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t filter_comma_delimited( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Track depth to identify root-level commas (depth 0) - int depth = 0; - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_root_comma_pos = 0; - uint32_t root_comma_count = 0; - - for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { - uint32_t idx = parser.structural_indexes[i]; - uint8_t c = parser.buf[idx]; - - switch (c) { - case '{': case '[': - depth++; - parser.structural_indexes[write_idx++] = idx; - break; - case '}': case ']': - depth--; - parser.structural_indexes[write_idx++] = idx; - break; - case ',': - if (depth == 0) { - // Root-level comma = document boundary, skip it - last_root_comma_pos = idx; - root_comma_count++; - continue; - } - parser.structural_indexes[write_idx++] = idx; - break; - default: - // Colons, scalars, etc. - parser.structural_indexes[write_idx++] = idx; - break; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { return 0; } - - if (is_final) { - // Final batch: use standard boundary detection on filtered indexes - return find_next_document_index(parser); - } - - // Partial batch: need to find complete documents only. - // A document ending with a root comma is complete. - if (root_comma_count == 0) { - // No root commas found; we cannot confirm any document is complete. - // The whole batch might be one incomplete document. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least one root comma. Documents before the last comma are complete. - next_batch_start = last_root_comma_pos + 1; - - // Find the structural index cutoff: keep only structurals < last_root_comma_pos - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_root_comma_pos) { - keep_count = i; - break; - } - } - - if (keep_count == 0) { return 0; } - - // Use standard boundary detection on the complete portion - parser.n_structural_indexes = keep_count; - return find_next_document_index(parser); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H -/* end file generic/stage1/find_next_document_index.h for arm64 */ -/* including generic/stage1/json_minifier.h for arm64: #include */ -/* begin file generic/stage1/json_minifier.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage1 { - -class json_minifier { -public: - template - static error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept; - -private: - simdjson_inline json_minifier(uint8_t *_dst) - : dst{_dst} - {} - template - simdjson_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_warn_unused simdjson_inline error_code finish(uint8_t *dst_start, size_t &dst_len); - json_scanner scanner{}; - uint8_t *dst; -}; - -simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { - uint64_t mask = block.whitespace(); - dst += in.compress(mask, dst); -} - -simdjson_warn_unused simdjson_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { - error_code error = scanner.finish(); - if (error) { dst_len = 0; return error; } - dst_len = dst - dst_start; - return SUCCESS; -} - -template<> -simdjson_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - simd::simd8x64 in_2(block_buf+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1); - this->next(in_2, block_2); - reader.advance(); -} - -template<> -simdjson_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - json_block block_1 = scanner.next(in_1); - this->next(block_buf, block_1); - reader.advance(); -} - -template -error_code json_minifier::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept { - buf_block_reader reader(buf, len); - json_minifier minifier(dst); - - // Index the first n-1 blocks - while (reader.has_full_block()) { - minifier.step(reader.full_block(), reader); - } - - // Index the last (remainder) block, padded with spaces - uint8_t block[STEP_SIZE]; - size_t remaining_bytes = reader.get_remainder(block); - if (remaining_bytes > 0) { - // We do not want to write directly to the output stream. Rather, we write - // to a local buffer (for safety). - uint8_t out_block[STEP_SIZE]; - uint8_t * const guarded_dst{minifier.dst}; - minifier.dst = out_block; - minifier.step(block, reader); - size_t to_write = minifier.dst - out_block; - // In some cases, we could be enticed to consider the padded spaces - // as part of the string. This is fine as long as we do not write more - // than we consumed. - if(to_write > remaining_bytes) { to_write = remaining_bytes; } - memcpy(guarded_dst, out_block, to_write); - minifier.dst = guarded_dst + to_write; - } - return minifier.finish(dst, dst_len); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H -/* end file generic/stage1/json_minifier.h for arm64 */ -/* including generic/stage1/json_structural_indexer.h for arm64: #include */ -/* begin file generic/stage1/json_structural_indexer.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage1 { - -class bit_indexer { -public: - uint32_t *tail; - - simdjson_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} - -#if SIMDJSON_PREFER_REVERSE_BITS - /** - * ARM lacks a fast trailing zero instruction, but it has a fast - * bit reversal instruction and a fast leading zero instruction. - * Thus it may be profitable to reverse the bits (once) and then - * to rely on a sequence of instructions that call the leading - * zero instruction. - * - * Performance notes: - * The chosen routine is not optimal in terms of data dependency - * since zero_leading_bit might require two instructions. However, - * it tends to minimize the total number of instructions which is - * beneficial. - */ - simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { - int lz = leading_zeroes(rev_bits); - this->tail[i] = static_cast(idx) + lz; - rev_bits = zero_leading_bit(rev_bits, lz); - } -#else - /** - * Under recent x64 systems, we often have both a fast trailing zero - * instruction and a fast 'clear-lower-bit' instruction so the following - * algorithm can be competitive. - */ - - simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { - this->tail[i] = idx + trailing_zeroes(bits); - bits = clear_lowest_bit(bits); - } -#endif // SIMDJSON_PREFER_REVERSE_BITS - - template - simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { - write_index(idx, bits, START); - SIMDJSON_IF_CONSTEXPR (N > 1) { - write_indexes<(N-1>0?START+1:START), (N-1>=0?N-1:1)>(idx, bits); - } - return START+N; - } - - template - simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { - write_indexes(idx, bits); - SIMDJSON_IF_CONSTEXPR ((START+STEP) < END) { - if (simdjson_unlikely((START+STEP) < cnt)) { - write_indexes_stepped<(START+STEP(idx, bits, cnt); - } - } - return ((END-START) % STEP) == 0 ? END : (END-START) - ((END-START) % STEP) + STEP; - } - - // flatten out values in 'bits' assuming that they are are to have values of idx - // plus their position in the bitvector, and store these indexes at - // base_ptr[base] incrementing base as we go - // will potentially store extra values beyond end of valid bits, so base_ptr - // needs to be large enough to handle this - // - // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it - // will provide its own version of the code. -#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - simdjson_inline void write(uint32_t idx, uint64_t bits); -#else - simdjson_inline void write(uint32_t idx, uint64_t bits) { - // In some instances, the next branch is expensive because it is mispredicted. - // Unfortunately, in other cases, - // it helps tremendously. - if (bits == 0) - return; - - int cnt = static_cast(count_ones(bits)); - -#if SIMDJSON_PREFER_REVERSE_BITS - bits = reverse_bits(bits); -#endif -#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP - static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; -#else - static constexpr const int STEP = 4; -#endif - static constexpr const int STEP_UNTIL = 24; - - write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); - SIMDJSON_IF_CONSTEXPR (STEP_UNTIL < 64) { - if (simdjson_unlikely(STEP_UNTIL < cnt)) { - for (int i=STEP_UNTIL; itail += cnt; - } -#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -}; - -class json_structural_indexer { -public: - /** - * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. - * - * @param partial Setting the partial parameter to true allows the find_structural_bits to - * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If - * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. - */ - template - static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept; - -private: - simdjson_inline json_structural_indexer(uint32_t *structural_indexes); - template - simdjson_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_warn_unused simdjson_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); - - json_scanner scanner{}; - utf8_checker checker{}; - bit_indexer indexer; - uint64_t prev_structurals = 0; - uint64_t unescaped_chars_error = 0; -}; - -simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} - -// Skip the last character if it is partial -simdjson_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { - if (simdjson_unlikely(len < 3)) { - switch (len) { - case 2: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left - return len; - case 1: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - return len; - case 0: - return len; - } - } - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left - return len; -} - -// -// PERF NOTES: -// We pipe 2 inputs through these stages: -// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load -// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. -// 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. -// The output of step 1 depends entirely on this information. These functions don't quite use -// up enough CPU: the second half of the functions is highly serial, only using 1 execution core -// at a time. The second input's scans has some dependency on the first ones finishing it, but -// they can make a lot of progress before they need that information. -// 3. Step 1 does not use enough capacity, so we run some extra stuff while we're waiting for that -// to finish: utf-8 checks and generating the output from the last iteration. -// -// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all -// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough -// workout. -// -template -error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept { - if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } - // We guard the rest of the code so that we can assume that len > 0 throughout. - if (len == 0) { return EMPTY; } - if (is_streaming(partial)) { - len = trim_partial_utf8(buf, len); - // If you end up with an empty window after trimming - // the partial UTF-8 bytes, then chances are good that you - // have an UTF-8 formatting error. - if(len == 0) { return UTF8_ERROR; } - } - buf_block_reader reader(buf, len); - json_structural_indexer indexer(parser.structural_indexes.get()); - - // Read all but the last block - while (reader.has_full_block()) { - indexer.step(reader.full_block(), reader); - } - // Take care of the last block (will always be there unless file is empty which is - // not supposed to happen.) - uint8_t block[STEP_SIZE]; - const size_t remaining = parser._unpadded ? reader.get_remainder(block) : reader.get_remainder_padded(block); - if (simdjson_unlikely(remaining == 0)) { return UNEXPECTED_ERROR; } - indexer.step(block, reader); - return indexer.finish(parser, reader.block_index(), len, partial); -} - -template<> -simdjson_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block); - simd::simd8x64 in_2(block+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1, reader.block_index()); - this->next(in_2, block_2, reader.block_index()+64); - reader.advance(); -} - -template<> -simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block); - json_block block_1 = scanner.next(in_1); - this->next(in_1, block_1, reader.block_index()); - reader.advance(); -} - -simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { - uint64_t unescaped = in.lteq(0x1F); -#if SIMDJSON_UTF8VALIDATION - checker.check_next_input(in); -#endif - indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser - prev_structurals = block.structural_start(); - unescaped_chars_error |= block.non_quote_inside_string(unescaped); -} - -// Drops the unclosed string that ends a json_sequence batch and leaves its -// start in structural_indexes[n_structural_indexes]. The last index is its -// opening quote, unless the quote directly follows an RS: the scanner reads -// RS as a scalar byte, so it indexes the RS and not the quote. Keep the RS. -simdjson_inline void drop_json_sequence_unclosed_string(dom_parser_implementation &parser, size_t len) noexcept { - uint32_t quote = parser.structural_indexes[parser.n_structural_indexes - 1]; - if (parser.buf[quote] == '"') { - parser.n_structural_indexes--; - return; - } - while (quote < len && parser.buf[quote] != '"') { quote++; } - parser.structural_indexes[parser.n_structural_indexes] = quote; -} - -simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial) { - // Write out the final iteration's structurals - indexer.write(uint32_t(idx-64), prev_structurals); - error_code error = scanner.finish(); - // We deliberately break down the next expression so that it is - // human readable. - const bool should_we_exit = is_streaming(partial) ? - ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING - : (error != SUCCESS); // if partial is false, we must have SUCCESS - const bool have_unclosed_string = (error == UNCLOSED_STRING); - parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); - /*** - * The On-Demand API requires special padding. - * - * This is related to https://github.com/simdjson/simdjson/issues/906 - * Basically, we want to make sure that if the parsing continues beyond the last (valid) - * structural character, it quickly stops. - * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. - * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing - * continues, then it must be [,] or }. - * Suppose it is ] or }. We backtrack to the first character, what could it be that would - * not trigger an error? It could be ] or } but no, because you can't start a document that way. - * It can't be a comma, a colon or any simple value. So the only way we could continue is - * if the repeated character is [. But if so, the document must start with [. But if the document - * starts with [, it should end with ]. If we enforce that rule, then we would get - * ][[ which is invalid. - * - * This is illustrated with the test array_iterate_unclosed_error() on the following input: - * R"({ "a": [,,)" - **/ - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final - parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); - parser.structural_indexes[parser.n_structural_indexes + 2] = 0; - parser.next_structural_index = 0; - - // Bail out only once the count and sentinels above are set: - // document_stream::truncated_bytes() reads them even on error. - if (simdjson_unlikely(should_we_exit)) { return error; } - if (unescaped_chars_error) { return UNESCAPED_CHARS; } - - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - return EMPTY; - } - if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { - return UNEXPECTED_ERROR; - } - if (partial == stage1_mode::streaming_partial) { - // If we have an unclosed string, then the last structural - // will be the quote and we want to make sure to omit it. - if(have_unclosed_string) { - parser.n_structural_indexes--; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - // We truncate the input to the end of the last complete document (or zero). - const bool partial_scalar = !have_unclosed_string && ends_with_partial_scalar(parser, len); - const bool unfinished = ends_with_unfinished_document(parser); - auto new_structural_indexes = find_next_document_index(parser, partial_scalar || unfinished); - if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { - // The batch holds a single document, whose brackets balance but which - // cannot be valid JSON: it is not too big, it is invalid. - if (unfinished && find_next_document_index(parser, partial_scalar) != 0) { return TAPE_ERROR; } - if(parser.structural_indexes[0] == 0) { - // If the buffer is partial and we started at index 0 but the document is - // incomplete, it's too big to parse. - return CAPACITY; - } else { - // It is possible that the document could be parsed, we just had a lot - // of white space. - parser.n_structural_indexes = 0; - return EMPTY; - } - } - parser.n_structural_indexes = new_structural_indexes; - } else if (partial == stage1_mode::streaming_final) { - if(have_unclosed_string) { parser.n_structural_indexes--; } - // We truncate the input to the end of the last complete document (or zero). - // Because partial == stage1_mode::streaming_final, it means that we may - // silently ignore trailing garbage. Though it sounds bad, we do it - // deliberately because many people who have streams of JSON documents - // will truncate them for processing. E.g., imagine that you are uncompressing - // the data from a size file or receiving it in chunks from the network. You - // may not know where exactly the last document will be. Meanwhile the - // document_stream instances allow people to know the JSON documents they are - // parsing (see the iterator.source() method). - parser.n_structural_indexes = find_next_document_index(parser); - // We store the initial n_structural_indexes so that the client can see - // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, - // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, - // otherwise, it will copy some prior index. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - // This next line is critical, do not change it unless you understand what you are - // doing. - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - // We tolerate an unclosed string at the very end of the stream. Indeed, users - // often load their data in bulk without being careful and they want us to ignore - // the trailing garbage. - return EMPTY; - } - } else if (partial == stage1_mode::json_sequence_partial) { - // RFC 7464: use RS positions for batch boundaries - // A discarded unclosed string also caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - // The next batch starts at an RS, which the On-Demand API never reads as a - // string, a container or a separator. But an invalid last document may make - // it skip that RS as a value: repeat it, so that the index past it is not - // the next record, which can be a string that does not close in this batch. - parser.structural_indexes[parser.n_structural_indexes + 1] = next_batch_start; - } else if (partial == stage1_mode::json_sequence_final) { - // RFC 7464: final batch, last document extends to EOF - // As above: a discarded unclosed string caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); - // The filter compacted structural_indexes in place and restored the EOF - // sentinel past the compacted end, so the copy below is either the start - // of a truncated document or len, as in streaming_final. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } else if (partial == stage1_mode::comma_delimited_partial) { - // Comma-delimited: filter root-level commas, use comma positions for batch boundaries - if(have_unclosed_string) { - parser.n_structural_indexes--; - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - } else if (partial == stage1_mode::comma_delimited_final) { - // Comma-delimited: final batch, last document extends to EOF - if(have_unclosed_string) { parser.n_structural_indexes--; } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } - checker.check_eof(); - return checker.errors(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. -#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H -/* end file generic/stage1/json_structural_indexer.h for arm64 */ -/* including generic/stage1/utf8_validator.h for arm64: #include */ -/* begin file generic/stage1/utf8_validator.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage1 { - -/** - * Validates that the string is actual UTF-8. - */ -template -bool generic_validate_utf8(const uint8_t * input, size_t length) { - checker c{}; - buf_block_reader<64> reader(input, length); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - c.check_next_input(in); - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - c.check_next_input(in); - reader.advance(); - c.check_eof(); - return c.errors() == error_code::SUCCESS; -} - -bool generic_validate_utf8(const char * input, size_t length) { - return generic_validate_utf8(reinterpret_cast(input),length); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H -/* end file generic/stage1/utf8_validator.h for arm64 */ -/* end file generic/stage1/amalgamated.h for arm64 */ -/* including generic/stage2/amalgamated.h for arm64: #include */ -/* begin file generic/stage2/amalgamated.h for arm64 */ -// Stuff other things depend on -/* including generic/stage2/base.h for arm64: #include */ -/* begin file generic/stage2/base.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage2 { - -class json_iterator; -struct tape_builder; -struct tape_writer; - -} // namespace stage2 -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H -/* end file generic/stage2/base.h for arm64 */ -/* including generic/stage2/tape_writer.h for arm64: #include */ -/* begin file generic/stage2/tape_writer.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage2 { - -struct tape_writer { - /** The next place to write to tape */ - uint64_t *next_tape_loc; - - /** Write a signed 64-bit value to tape. */ - simdjson_inline void append_s64(int64_t value) noexcept; - - /** Write an unsigned 64-bit value to tape. */ - simdjson_inline void append_u64(uint64_t value) noexcept; - - /** Write a double value to tape. */ - simdjson_inline void append_double(double value) noexcept; - - /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ - simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; - - /** - * Append a tape entry (an 8-bit type,and 56 bits worth of value). - */ - simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; - - /** - * Skip the current tape entry without writing. - * - * Used to skip the start of the container, since we'll come back later to fill it in when the - * container ends. - */ - simdjson_inline void skip() noexcept; - - /** - * Skip the number of tape entries necessary to write a large u64 or i64. - */ - simdjson_inline void skip_large_integer() noexcept; - - /** - * Skip the number of tape entries necessary to write a double. - */ - simdjson_inline void skip_double() noexcept; - - /** - * Write a value to a known location on tape. - * - * Used to go back and write out the start of a container after the container ends. - */ - simdjson_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; - -private: - /** - * Append both the tape entry, and a supplementary value following it. Used for types that need - * all 64 bits, such as double and uint64_t. - */ - template - simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; -}; // struct tape_writer - -simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { - append2(0, value, internal::tape_type::INT64); -} - -simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { - append(0, internal::tape_type::UINT64); - *next_tape_loc = value; - next_tape_loc++; -} - -/** Write a double value to tape. */ -simdjson_inline void tape_writer::append_double(double value) noexcept { - append2(0, value, internal::tape_type::DOUBLE); -} - -simdjson_inline void tape_writer::skip() noexcept { - next_tape_loc++; -} - -simdjson_inline void tape_writer::skip_large_integer() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::skip_double() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { - *next_tape_loc = val | ((uint64_t(char(t))) << 56); - next_tape_loc++; -} - -template -simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { - append(val, t); - static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); - memcpy(next_tape_loc, &val2, sizeof(val2)); - next_tape_loc++; -} - -simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { - tape_loc = val | ((uint64_t(char(t))) << 56); -} - -simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { - // Write to string buffer: [4-byte LE length][digits][null] - uint32_t str_len = uint32_t(len); - memcpy(string_buf, &str_len, sizeof(uint32_t)); - memcpy(string_buf + sizeof(uint32_t), src, len); - string_buf[sizeof(uint32_t) + len] = 0; - // Tape entry: offset into string buffer - // The caller must set the offset relative to doc.string_buf base - append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches - string_buf += sizeof(uint32_t) + len + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H -/* end file generic/stage2/tape_writer.h for arm64 */ -/* including generic/stage2/logger.h for arm64: #include */ -/* begin file generic/stage2/logger.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - - -// This is for an internal-only stage 2 specific logger. -// Set LOG_ENABLED = true to log what stage 2 is doing! -namespace simdjson { -namespace arm64 { -namespace { -namespace logger { - - static constexpr const char * DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; - -#if SIMDJSON_VERBOSE_LOGGING - static constexpr const bool LOG_ENABLED = true; -#else - static constexpr const bool LOG_ENABLED = false; -#endif - static constexpr const int LOG_EVENT_LEN = 20; - static constexpr const int LOG_BUFFER_LEN = 30; - static constexpr const int LOG_SMALL_BUFFER_LEN = 10; - static constexpr const int LOG_INDEX_LEN = 5; - - static int log_depth; // Not threadsafe. Log only. - - // Helper to turn unprintable or newline characters into spaces - static simdjson_inline char printable_char(char c) { - if (c >= 0x20) { - return c; - } else { - return ' '; - } - } - - // Print the header and set up log_start - static simdjson_inline void log_start() { - if (LOG_ENABLED) { - log_depth = 0; - printf("\n"); - printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); - printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN+2, DASHES, LOG_BUFFER_LEN+2, DASHES, LOG_SMALL_BUFFER_LEN+2, DASHES, 5+2, DASHES); - } - } - - simdjson_unused static simdjson_inline void log_string(const char *message) { - if (LOG_ENABLED) { - printf("%s\n", message); - } - } - - // Logs a single line from the stage 2 DOM parser - template - static simdjson_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { - if (LOG_ENABLED) { - printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); - auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; - auto next_index = structurals.next_structural; - auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); - auto next = &structurals.buf[*next_index]; - { - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i=0;i */ -/* begin file generic/stage2/json_iterator.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage2 { - -class json_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - uint32_t depth{0}; - - /** - * Walk the JSON document. - * - * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as - * the first parameter; some callbacks have other parameters as well: - * - * - visit_document_start() - at the beginning. - * - visit_document_end() - at the end (if things were successful). - * - * - visit_array_start() - at the start `[` of a non-empty array. - * - visit_array_end() - at the end `]` of a non-empty array. - * - visit_empty_array() - when an empty array is encountered. - * - * - visit_object_end() - at the start `]` of a non-empty object. - * - visit_object_start() - at the end `]` of a non-empty object. - * - visit_empty_object() - when an empty object is encountered. - * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is - * guaranteed to point at the first quote of the string (`"key"`). - * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. - * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. - * - * - increment_count(iter) - each time a value is found in an array or object. - */ - template - simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; - - /** - * Create an iterator capable of walking a JSON document. - * - * The document must have already passed through stage 1. - */ - simdjson_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); - - /** - * Look at the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *peek(const uint32_t *batch_end = nullptr) const noexcept; - /** - * Advance to the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *advance(const uint32_t *batch_end = nullptr) noexcept; - /** - * Get the remaining length of the document, from the start of the current token. - */ - simdjson_inline size_t remaining_len() const noexcept; - /** - * Check if we are at the end of the document. - * - * If this is true, there are no more tokens. - */ - simdjson_inline bool at_eof() const noexcept; - /** - * Check if we are at the beginning of the document. - */ - simdjson_inline bool at_beginning() const noexcept; - simdjson_inline uint8_t last_structural() const noexcept; - - /** - * Log that a value has been found. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_value(const char *type) const noexcept; - /** - * Log the start of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_start_value(const char *type) const noexcept; - /** - * Log the end of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_end_value(const char *type) const noexcept; - /** - * Log an error. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_error(const char *error) const noexcept; - - template - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; - template - simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; -}; - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { - logger::log_start(); - - // - // Start the document - // - if (at_eof()) { return EMPTY; } - // Only used when BOUNDED (see advance()); a local, so that it can stay in a - // register, and vanishes from the other walks. - const uint32_t *const batch_end = &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - log_start_value("document"); - SIMDJSON_TRY( visitor.visit_document_start(*this) ); - - // - // Read first value - // - { - auto value = advance(batch_end); - - // Make sure the outer object or array is closed before continuing; otherwise, there are ways we - // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 - if (!STREAMING) { - switch (*value) { - case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; - case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; - } - } - - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; - } - } - goto document_end; - -// -// Object parser states -// -object_begin: - log_start_value("object"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = false; - SIMDJSON_TRY( visitor.visit_object_start(*this) ); - - { - auto key = advance(batch_end); - if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.increment_count(*this) ); - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - -object_field: - if (simdjson_unlikely( (*advance(batch_end) != ':') )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -object_continue: - switch (*advance(batch_end)) { - case ',': - SIMDJSON_TRY( visitor.increment_count(*this) ); - { - auto key = advance(batch_end); - if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - goto object_field; - case '}': log_end_value("object"); SIMDJSON_TRY( visitor.visit_object_end(*this) ); goto scope_end; - default: log_error("No comma between object fields"); return TAPE_ERROR; - } - -scope_end: - depth--; - if (depth == 0) { goto document_end; } - if (dom_parser.is_array[depth]) { goto array_continue; } - goto object_continue; - -// -// Array parser states -// -array_begin: - log_start_value("array"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = true; - SIMDJSON_TRY( visitor.visit_array_start(*this) ); - SIMDJSON_TRY( visitor.increment_count(*this) ); - -array_value: - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -array_continue: - switch (*advance(batch_end)) { - case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; - case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; - default: log_error("Missing comma between array values"); return TAPE_ERROR; - } - -document_end: - log_end_value("document"); - SIMDJSON_TRY( visitor.visit_document_end(*this) ); - - dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); - - // If we didn't make it to the end, it's an error - if ( !STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes ) { - log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); - return TAPE_ERROR; - } - - return SUCCESS; - -} // walk_document() - -simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { -} - -// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires -// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). -// -// When BOUNDED (see dom_parser_implementation::_bounded_batch), a document may -// not run past batch_end, one past the last structural index of the batch: -// past it, we return the same dummy, and walk_document fails cleanly. -template -simdjson_inline const uint8_t *json_iterator::peek(const uint32_t *batch_end) const noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -template -simdjson_inline const uint8_t *json_iterator::advance(const uint32_t *batch_end) noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural++); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -simdjson_inline size_t json_iterator::remaining_len() const noexcept { - return dom_parser.len - *(next_structural-1); -} - -simdjson_inline bool json_iterator::at_eof() const noexcept { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; -} -simdjson_inline bool json_iterator::at_beginning() const noexcept { - return next_structural == dom_parser.structural_indexes.get(); -} -simdjson_inline uint8_t json_iterator::last_structural() const noexcept { - return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; -} - -simdjson_inline void json_iterator::log_value(const char *type) const noexcept { - logger::log_line(*this, "", type, ""); -} - -simdjson_inline void json_iterator::log_start_value(const char *type) const noexcept { - logger::log_line(*this, "+", type, ""); - if (logger::LOG_ENABLED) { logger::log_depth++; } -} - -simdjson_inline void json_iterator::log_end_value(const char *type) const noexcept { - if (logger::LOG_ENABLED) { logger::log_depth--; } - logger::log_line(*this, "-", type, ""); -} - -simdjson_inline void json_iterator::log_error(const char *error) const noexcept { - logger::log_line(*this, "", "ERROR", error); -} - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { - switch (*value) { - case '"': return visitor.visit_root_string(*this, value); - case 't': return visitor.visit_root_true_atom(*this, value); - case 'f': return visitor.visit_root_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_root_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_root_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_root_inf_atom(*this, value); -#else - case 'n': return visitor.visit_root_null_atom(*this, value); -#endif - case '-': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return visitor.visit_root_number(*this, value); - default: - log_error("Document starts with a non-value character"); - return TAPE_ERROR; - } -} -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { - // Use the fact that most scalars are going to be either strings or numbers. - if(*value == '"') { - return visitor.visit_string(*this, value); - } else if (((*value - '0') < 10) || (*value == '-')) { - return visitor.visit_number(*this, value); - } - // true, false, null are uncommon. - switch (*value) { - case 't': return visitor.visit_true_atom(*this, value); - case 'f': return visitor.visit_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_inf_atom(*this, value); -#else - case 'n': return visitor.visit_null_atom(*this, value); -#endif - default: - log_error("Non-value found when value was expected!"); - return TAPE_ERROR; - } -} - -} // namespace stage2 -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H -/* end file generic/stage2/json_iterator.h for arm64 */ -/* including generic/stage2/stringparsing.h for arm64: #include */ -/* begin file generic/stage2/stringparsing.h for arm64 */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace arm64 { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - // Decode adjacent Unicode escapes without returning to the - // quote-and-backslash scanner between code points. - do { - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } while (src[0] == '\\' && src[1] == 'u'); - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -/** - * Bounds-safe variant of parse_string for input buffers that are NOT padded to - * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input - * byte (buf + len). It behaves exactly like parse_string while we are at least - * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays - * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few - * remaining bytes into a space-padded scratch buffer and finishes with the - * regular parse_string. This keeps the delicate escape/Unicode handling in one - * place (the proven parse_string) rather than duplicating it. - * - * Correctness relies on stage 1 having validated the string, i.e. there is an - * unescaped closing quote within [src, buf_end); that quote is therefore inside - * the copied scratch, so parse_string finds it without running off the scratch. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { - // Far from the end: identical to parse_string's loop. The guard uses - // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past - // buf_end; escape/Unicode look-aheads read within the string (before the - // closing quote, which is < buf_end), so they are in bounds here too. - // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: - // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate - // following bytes (+6 from the '\'), then (if it sees \u) another - // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the - // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the - // chunk start. The +12 margin ensures that even on kernels where - // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays - // in-bounds. The scratch fallback (3*PAD) is already safe. - while (src + SIMDJSON_PADDING + 12 <= buf_end) { - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - if (bs_quote.has_quote_first()) { - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - if (escape_char == 'u') { - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - // Within the final SIMDJSON_PADDING bytes: copy what remains into a - // space-padded scratch (spaces are neither quote nor backslash, so they do not - // disturb matching) and let the regular parser finish from there. The closing - // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in - // the chunk starting at some offset <= remaining and reads at most - // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. - // We size at 3x for a comfortable margin (the unicode look-ahead reads a few - // extra bytes past an escape). - uint8_t scratch[SIMDJSON_PADDING * 3]; - const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING - std::memset(scratch, ' ', sizeof(scratch)); - std::memcpy(scratch, src, remaining); - return parse_string(scratch, dst, allow_replacement); -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for arm64 */ -/* including generic/stage2/tape_builder.h for arm64: #include */ -/* begin file generic/stage2/tape_builder.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage2 { - -template -struct tape_builder_impl { - /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; - /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator &iter) noexcept; - - /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator &iter) noexcept; - /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator &iter) noexcept; - /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator &iter) noexcept; - - /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator &iter) noexcept; - /** - * Called when a key in a field is encountered. - * - * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array - * will be called after this with the field value. - */ - simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; - /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator &iter) noexcept; - /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator &iter) noexcept; - - /** - * Called when a string, number, boolean or null is found. - */ - simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; - /** - * Called when a string, number, boolean or null is found at the top level of a document (i.e. - * when there is no array or object and the entire document is a single string, number, boolean or - * null. - * - * This is separate from primitive() because simdjson's normal primitive parsing routines assume - * there is at least one more token after the value, which is only true in an array or object. - */ - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - -#if SIMDJSON_ENABLE_NAN_INF - simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, - // this returns a tape error on failure. - simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; -#endif - - /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; - - /** Next location to write to tape */ - tape_writer tape; -public: - simdjson_inline tape_builder_impl(dom::document &doc) noexcept; -private: - /** Next write location in the string buf for stage 2 parsing */ - uint8_t *current_string_buf_loc; - - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - simdjson_inline void start_container(json_iterator &iter) noexcept; - simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder_impl - -// Thin, non-templated entry so each architecture's stage2() keeps calling -// tape_builder::parse_document unchanged. It chooses the bounds-safe -// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the -// choice is a compile-time constant inside the walk: the padded path carries no -// extra branch or load (see tape_builder_impl::visit_string). -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } - - // The stage 2 entry points of dom::document_stream, which picks one per batch - // (see internal::dom_parser_implementation::stage2_next_function). Out of - // line, so that stage2_next() can share the code. - simdjson_warn_unused static simdjson_never_inline error_code parse_next_document( - internal::dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - return parse_document(static_cast(dom_parser), doc); - } - - // A document of a stream batch that stage 2 must bound to its structural - // indexes (see internal::dom_parser_implementation::_bounded_batch). - simdjson_warn_unused static simdjson_never_inline error_code parse_next_bounded_document( - internal::dom_parser_implementation &_dom_parser, dom::document &doc) noexcept { - dom_parser_implementation &dom_parser = static_cast(_dom_parser); - dom_parser.doc = &doc; - json_iterator iter(dom_parser, dom_parser.next_structural_index); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } -}; - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_root_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { - constexpr uint32_t start_tape_index = 0; - tape.append(start_tape_index, internal::tape_type::ROOT); - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { - return visit_string(iter, key, true); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 - return SUCCESS; -} - -template -simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { - iter.log_value(key ? "key" : "string"); - uint8_t *dst = on_start_string(iter); - // We do not allow replacement when the escape characters are invalid. - // UNPADDED is a compile-time constant chosen once per document by - // tape_builder::parse_document, so the padded build instantiates only the - // plain parse_string call below -- no runtime branch and no flag load. - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); - } else { - dst = stringparsing::parse_string(value+1, dst, false); - } - if (dst == nullptr) { - iter.log_error("Invalid escape in string"); - return STRING_ERROR; - } - on_end_string(dst); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { - return visit_string(iter, value); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("number"); - const uint8_t *num = value; - std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - // numberparsing reads ahead in 8-byte blocks for floats - // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a - // number whose digits reach the final bytes of an unpadded buffer would read - // past it. *(next_structural) is the offset of the token following this - // number, hence an upper bound on where the digits end; when that is within - // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail - // (mirroring visit_root_number). This fires only for numbers near the end. - if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { - const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document - copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, rl); - std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); - num = copy.get(); - } - } - error_code err = numberparsing::parse_number(num, tape); - if (simdjson_unlikely(err == BIGINT_ERROR && - iter.dom_parser._number_as_string)) { - // Write big integer to string buffer using the same format as strings. - // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = num; - if (*p == '-') p++; - while (numberparsing::is_digit(*p)) p++; - // The digit run must be terminated by a structural or whitespace character; otherwise the - // token is malformed (e.g. "123456789123456789123x"). - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - num); - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); - uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, num, len); - dst += len; - on_end_string(dst); - return SUCCESS; - } - return err; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { - // - // We need to make a copy to make sure that the string is space terminated. - // This is not about padding the input, which should already padded up - // to len + SIMDJSON_PADDING. However, we have no control at this stage - // on how the padding was done. What if the input string was padded with nulls? - // It is quite common for an input string to have an extra null character (C string). - // We do not want to allow 9\0 (where \0 is the null character) inside a JSON - // document, but the string "9\0" by itself is fine. So we make a copy and - // pad the input with spaces when we know that there is just one input element. - // This copy is relatively expensive, but it will almost never be called in - // practice unless you are in the strange scenario where you have many JSON - // documents made of single atoms. - // - // In a stream, the input goes on with other documents: copy up to the next - // structural only, not to the end of the batch. - const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); - std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, len); - std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); - error_code error = visit_number(iter, copy.get()); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated - // token at the very end of an unpadded buffer would over-read. Use the - // length-aware form there (the root variant already does this). - const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) - : atomparsing::is_valid_true_atom(value); - if (!ok) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) - : atomparsing::is_valid_false_atom(value); - if (!ok) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) - : atomparsing::is_valid_null_atom(value); - if (!ok) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -#if SIMDJSON_ENABLE_NAN_INF -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - // For unpadded input use the length-aware validator so the 'infinity'-style - // 8-byte compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) - : atomparsing::is_valid_nan_atom(value); - if (!ok) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. - // For unpadded input use the length-aware validator so the 'infinity' 8-byte - // compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) - : atomparsing::is_valid_inf_atom(value); - if (!ok) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure - if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -// private: - -template -simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { - return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - auto start_index = next_tape_index(iter); - tape.append(start_index+2, start); - tape.append(start_index, end); - return SUCCESS; -} - -template -simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); - iter.dom_parser.open_containers[iter.depth].count = 0; - tape.skip(); // We don't actually *write* the start element until the end. -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - // Write the ending tape element, pointing at the start location - const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; - tape.append(start_tape_index, end); - // Write the start tape element, pointing at the end location (and including count) - // count can overflow if it exceeds 24 bits... so we saturate - // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). - const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; - const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); - return SUCCESS; -} - -template -simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { - // we advance the point, accounting for the fact that we have a NULL termination - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); - return current_string_buf_loc + sizeof(uint32_t); -} - -template -simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { - uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); - // TODO check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); - // NULL termination is still handy if you expect all your strings to - // be NULL terminated? It comes at a small cost - *dst = 0; - current_string_buf_loc = dst + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H -/* end file generic/stage2/tape_builder.h for arm64 */ -/* end file generic/stage2/amalgamated.h for arm64 */ - -#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -namespace simdjson { namespace arm64 { namespace { namespace stage1 { - -// The generic bit_indexer::write emits the structural indexes in groups of -// four (up to 24), so a block with five set bits computes and stores eight -// indexes, three of them wasted. Typical JSON has four to eight structural -// characters per 64-byte block. This version writes the first four indexes -// unconditionally and then continues by groups of two, which cuts the wasted -// work on such blocks at the cost of one more branch for dense blocks. Both -// versions fall back to the same scalar loop past 24 indexes. -simdjson_inline void bit_indexer::write(uint32_t idx, uint64_t bits) { - if (bits == 0) { return; } - - const int cnt = static_cast(count_ones(bits)); -#if SIMDJSON_PREFER_REVERSE_BITS - bits = reverse_bits(bits); -#endif - write_indexes<0, 4>(idx, bits); - if (simdjson_unlikely(4 < cnt)) { - write_indexes_stepped<4, 24, 2>(idx, bits, cnt); - } - if (simdjson_unlikely(24 < cnt)) { - for (int i = 24; i < cnt; ++i) { - write_index(idx, bits, i); - } - } - this->tail += cnt; -} - -}}}} // namespace simdjson::arm64::(anonymous)::stage1 - -// -// Stage 1 -// -namespace simdjson { -namespace arm64 { - -simdjson_warn_unused error_code implementation::create_dom_parser_implementation( - size_t capacity, - size_t max_depth, - std::unique_ptr& dst -) const noexcept { - dst.reset( new (std::nothrow) dom_parser_implementation() ); - if (!dst) { return MEMALLOC; } - if (auto err = dst->set_capacity(capacity)) - return err; - if (auto err = dst->set_max_depth(max_depth)) - return err; - return SUCCESS; -} - -namespace { - -using namespace simd; - -simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - const uint8x16_t op_table = simd8( - 0xff, 0, ',', ':', 0, '[', ']', '{', '}', 0, 0, 0, 0, 0, 0, 0 - ); - const uint8x16_t ws_table = simd8( - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0, 0, 0xff, 0, 0 - ); - - const uint8x16_t d0_0 = in.chunks[0]; - const uint8x16_t d0_1 = in.chunks[1]; - const uint8x16_t d0_2 = in.chunks[2]; - const uint8x16_t d0_3 = in.chunks[3]; - - const uint8x16_t match_op_0 = vceqq_u8(vqtbl1q_u8(op_table, vshrq_n_u8(vaddq_u8(d0_0, vdupq_n_u8(3)), 4)), d0_0); - const uint8x16_t match_op_1 = vceqq_u8(vqtbl1q_u8(op_table, vshrq_n_u8(vaddq_u8(d0_1, vdupq_n_u8(3)), 4)), d0_1); - const uint8x16_t match_op_2 = vceqq_u8(vqtbl1q_u8(op_table, vshrq_n_u8(vaddq_u8(d0_2, vdupq_n_u8(3)), 4)), d0_2); - const uint8x16_t match_op_3 = vceqq_u8(vqtbl1q_u8(op_table, vshrq_n_u8(vaddq_u8(d0_3, vdupq_n_u8(3)), 4)), d0_3); - - const uint8x16_t match_ws_0 = vqtbx1q_u8(vceqq_u8(d0_0, vdupq_n_u8(' ')), ws_table, d0_0); - const uint8x16_t match_ws_1 = vqtbx1q_u8(vceqq_u8(d0_1, vdupq_n_u8(' ')), ws_table, d0_1); - const uint8x16_t match_ws_2 = vqtbx1q_u8(vceqq_u8(d0_2, vdupq_n_u8(' ')), ws_table, d0_2); - const uint8x16_t match_ws_3 = vqtbx1q_u8(vceqq_u8(d0_3, vdupq_n_u8(' ')), ws_table, d0_3); - - const uint8x16_t bit_mask = simd8( - 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 - ); - - uint8x16_t op_sum0 = vpaddq_u8(vandq_u8(match_op_0, bit_mask), vandq_u8(match_op_1, bit_mask)); - uint8x16_t ws_sum0 = vpaddq_u8(vandq_u8(match_ws_0, bit_mask), vandq_u8(match_ws_1, bit_mask)); - uint8x16_t op_sum1 = vpaddq_u8(vandq_u8(match_op_2, bit_mask), vandq_u8(match_op_3, bit_mask)); - uint8x16_t ws_sum1 = vpaddq_u8(vandq_u8(match_ws_2, bit_mask), vandq_u8(match_ws_3, bit_mask)); - op_sum0 = vpaddq_u8(op_sum0, op_sum1); - ws_sum0 = vpaddq_u8(ws_sum0, ws_sum1); - op_sum0 = vpaddq_u8(op_sum0, op_sum0); - ws_sum0 = vpaddq_u8(ws_sum0, ws_sum0); - const uint64_t op = vgetq_lane_u64(vreinterpretq_u64_u8(op_sum0), 0); - const uint64_t whitespace = vgetq_lane_u64(vreinterpretq_u64_u8(ws_sum0), 0); - - return { whitespace, op }; -} - -simdjson_inline bool is_ascii(const simd8x64& input) { - simd8 bits = input.reduce_or(); - return bits.max_val() < 0x80u; -} - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be >= 0x80 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be >= 0x80 - return is_third_byte | is_fourth_byte; -} - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -// -// Stage 2 -// - -// -// Implementation-specific overrides -// -namespace simdjson { -namespace arm64 { - -simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { - return arm64::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); -} - -simdjson_flatten simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, stage1_mode streaming) noexcept { - this->buf = _buf; - this->len = _len; - return arm64::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); -} - -simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return arm64::stage1::generic_validate_utf8(buf,len); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_next_document(*this, _doc); -} - -internal::dom_parser_implementation::stage2_next_function_t dom_parser_implementation::stage2_next_function(bool bounded) const noexcept { - return bounded ? stage2::tape_builder::parse_next_bounded_document : stage2::tape_builder::parse_next_document; -} - -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { - return arm64::stringparsing::parse_string(src, dst, allow_replacement); -} - -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { - return arm64::stringparsing::parse_wobbly_string(src, dst); -} - -simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { - auto error = stage1(_buf, _len, stage1_mode::regular); - if (error) { return error; } - return stage2(_doc); -} - -} // namespace arm64 -} // namespace simdjson - -/* including simdjson/arm64/end.h: #include */ -/* begin file simdjson/arm64/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT -/* undefining SIMDJSON_IMPLEMENTATION from "arm64" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/arm64/end.h */ - -#endif // SIMDJSON_SRC_ARM64_CPP -/* end file arm64.cpp */ -#endif -#if SIMDJSON_IMPLEMENTATION_HASWELL -/* including haswell.cpp: #include */ -/* begin file haswell.cpp */ -#ifndef SIMDJSON_SRC_HASWELL_CPP -#define SIMDJSON_SRC_HASWELL_CPP - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* including simdjson/haswell.h: #include */ -/* begin file simdjson/haswell.h */ -#ifndef SIMDJSON_HASWELL_H -#define SIMDJSON_HASWELL_H - -/* including simdjson/haswell/begin.h: #include "simdjson/haswell/begin.h" */ -/* begin file simdjson/haswell/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "haswell" */ -#define SIMDJSON_IMPLEMENTATION haswell - -/* including simdjson/haswell/base.h: #include "simdjson/haswell/base.h" */ -/* begin file simdjson/haswell/base.h */ -#ifndef SIMDJSON_HASWELL_BASE_H -#define SIMDJSON_HASWELL_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_HASWELL -namespace simdjson { -/** - * Implementation for Haswell (Intel AVX2). - */ -namespace haswell { - -class implementation; - -namespace { -namespace simd { -template struct simd8; -template struct simd8x64; -} // namespace simd -} // unnamed namespace - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_BASE_H -/* end file simdjson/haswell/base.h */ -/* including simdjson/haswell/intrinsics.h: #include "simdjson/haswell/intrinsics.h" */ -/* begin file simdjson/haswell/intrinsics.h */ -#ifndef SIMDJSON_HASWELL_INTRINSICS_H -#define SIMDJSON_HASWELL_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - * e.g., if __AVX2__ is set... in turn, we normally set these - * macros by compiling against the corresponding architecture - * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole - * software with these advanced instructions. In simdjson, we - * want to compile the whole program for a generic target, - * and only target our specific kernels. As a workaround, - * we directly include the needed headers. These headers would - * normally guard against such usage, but we carefully included - * (or ) before, so the headers - * are fooled. - */ -#include // for _blsr_u64 -#include // for __lzcnt64 -#include // for most things (AVX2, AVX512, _popcnt64) -#include -#include -#include -#include -#include // for _mm_clmulepi64_si128 -// unfortunately, we may not get _blsr_u64, but, thankfully, clang -// has it as a macro. -#ifndef _blsr_u64 -// we roll our own -#define _blsr_u64(n) ((n - 1) & n) -#endif // _blsr_u64 -#endif // SIMDJSON_CLANG_VISUAL_STUDIO - -static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for haswell kernel."); - -#endif // SIMDJSON_HASWELL_INTRINSICS_H -/* end file simdjson/haswell/intrinsics.h */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_HASWELL -// We enable bmi2 only if LLVM/clang is used, because GCC may not -// make good use of it. See https://github.com/simdjson/simdjson/pull/2243 -#if defined(__clang__) -SIMDJSON_TARGET_REGION("avx2,bmi,bmi2,pclmul,lzcnt,popcnt") -#else -SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt,popcnt") -#endif -#endif - -/* including simdjson/haswell/bitmanipulation.h: #include "simdjson/haswell/bitmanipulation.h" */ -/* begin file simdjson/haswell/bitmanipulation.h */ -#ifndef SIMDJSON_HASWELL_BITMANIPULATION_H -#define SIMDJSON_HASWELL_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmask.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return (int)_tzcnt_u64(input_num); -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - //////// - // You might expect the next line to be equivalent to - // return (int)_tzcnt_u64(input_num); - // but the generated code differs and might be less efficient? - //////// - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return _blsr_u64(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { - return int(_lzcnt_u64(input_num)); -} - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num);// Visual Studio wants two underscores -} -#else -simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); -} -#endif - - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_BITMANIPULATION_H -/* end file simdjson/haswell/bitmanipulation.h */ -/* including simdjson/haswell/bitmask.h: #include "simdjson/haswell/bitmask.h" */ -/* begin file simdjson/haswell/bitmask.h */ -#ifndef SIMDJSON_HASWELL_BITMASK_H -#define SIMDJSON_HASWELL_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { - // There should be no such thing with a processor supporting avx2 - // but not clmul. - __m128i all_ones = _mm_set1_epi8('\xFF'); - __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); - return _mm_cvtsi128_si64(result); -} - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_BITMASK_H -/* end file simdjson/haswell/bitmask.h */ -/* including simdjson/haswell/numberparsing_defs.h: #include "simdjson/haswell/numberparsing_defs.h" */ -/* begin file simdjson/haswell/numberparsing_defs.h */ -#ifndef SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H -#define SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace numberparsing { - -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - // this actually computes *16* values so we are being wasteful. - const __m128i ascii0 = _mm_set1_epi8('0'); - const __m128i mul_1_10 = - _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); - const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); - const __m128i mul_1_10000 = - _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); - const __m128i input = _mm_sub_epi8( - _mm_loadu_si128(reinterpret_cast(chars)), ascii0); - const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); - const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); - const __m128i t3 = _mm_packus_epi32(t2, t2); - const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); - return _mm_cvtsi128_si32( - t4); // only captures the sum of the first 8 digits, drop the rest -} - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace haswell -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H -/* end file simdjson/haswell/numberparsing_defs.h */ -/* including simdjson/haswell/simd.h: #include "simdjson/haswell/simd.h" */ -/* begin file simdjson/haswell/simd.h */ -#ifndef SIMDJSON_HASWELL_SIMD_H -#define SIMDJSON_HASWELL_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m256i value; - - // Zero constructor - simdjson_inline base() : value{__m256i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m256i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m256i&() const { return this->value; } - simdjson_inline operator __m256i&() { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm256_or_si256(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm256_and_si256(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8: base> { - typedef uint32_t bitmask_t; - typedef uint64_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m256i _value) : base>(_value) {} - - friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm256_cmpeq_epi8(lhs, rhs); } - - static const int SIZE = sizeof(base::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return _mm256_alignr_epi8(*this, _mm256_permute2x128_si256(prev_chunk, *this, 0x21), 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m256i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return _mm256_movemask_epi8(*this); } - simdjson_inline bool any() const { return !_mm256_testz_si256(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { return _mm256_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm256_setzero_si256(); } - static simdjson_inline simd8 load(const T values[32]) { - return _mm256_loadu_si256(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m256i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[32]) const { return _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), *this); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm256_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm256_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm256_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 32 - count_ones(mask) bytes of the result are significant but 32 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint32_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in four steps, first 8 bytes and then second 8 bytes... - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits - uint8_t mask3 = uint8_t(mask >> 16); // ... - uint8_t mask4 = uint8_t(mask >> 24); // ... - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - __m256i shufmask = _mm256_set_epi64x(thintable_epi8[mask4], thintable_epi8[mask3], - thintable_epi8[mask2], thintable_epi8[mask1]); - // we increment by 0x08 the second half of the mask and so forth - shufmask = - _mm256_add_epi8(shufmask, _mm256_set_epi32(0x18181818, 0x18181818, - 0x10101010, 0x10101010, 0x08080808, 0x08080808, 0, 0)); - // this is the version "nearly pruned" - __m256i pruned = _mm256_shuffle_epi8(*this, shufmask); - // we still need to put the pieces back together. - // we compute the popcount of the first words: - int pop1 = BitsSetTable256mul2[mask1]; - int pop3 = BitsSetTable256mul2[mask3]; - - // then load the corresponding mask - // could be done with _mm256_loadu2_m128i but many standard libraries omit this intrinsic. - __m256i v256 = _mm256_castsi128_si256( - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8))); - __m256i compactmask = _mm256_insertf128_si256(v256, - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop3 * 8)), 1); - __m256i almostthere = _mm256_shuffle_epi8(pruned, compactmask); - // We just need to write out the result. - // This is the tricky bit that is hard to do - // if we want to return a SIMD register, since there - // is no single-instruction approach to recombine - // the two 128-bit lanes with an offset. - __m128i v128; - v128 = _mm256_castsi256_si128(almostthere); - _mm_storeu_si128( reinterpret_cast<__m128i *>(output), v128); - v128 = _mm256_extractf128_si256(almostthere, 1); - _mm_storeu_si128( reinterpret_cast<__m128i *>(output + 16 - count_ones(mask & 0xFFFF)), v128); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, - int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, - int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31 - ) : simd8(_mm256_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v16,v17,v18,v19,v20,v21,v22,v23, - v24,v25,v26,v27,v28,v29,v30,v31 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epi8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return _mm256_cmpgt_epi8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm256_cmpgt_epi8(other, *this); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, - uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, - uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31 - ) : simd8(_mm256_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v16,v17,v18,v19,v20,v21,v22,v23, - v24,v25,v26,v27,v28,v29,v30,v31 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm256_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm256_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epu8(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7-N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 2, "Haswell kernel should use two registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+32)} {} - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - uint32_t mask1 = uint32_t(mask); - uint32_t mask2 = uint32_t(mask >> 32); - this->chunks[0].compress(mask1, output); - this->chunks[1].compress(mask2, output + 32 - count_ones(mask1)); - return 64 - count_ones(mask); - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r_hi = this->chunks[1].to_bitmask(); - return r_lo | (r_hi << 32); - } - - simdjson_inline simd8 reduce_or() const { - return this->chunks[0] | this->chunks[1]; - } - - simdjson_inline simd8x64 bit_or(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] | mask, - this->chunks[1] | mask - ); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_SIMD_H -/* end file simdjson/haswell/simd.h */ -/* including simdjson/haswell/stringparsing_defs.h: #include "simdjson/haswell/stringparsing_defs.h" */ -/* begin file simdjson/haswell/stringparsing_defs.h */ -#ifndef SIMDJSON_HASWELL_STRINGPARSING_DEFS_H -#define SIMDJSON_HASWELL_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 15 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - // store to dest unconditionally - we can overwrite the bits we don't like later - v.store(dst); - return { - static_cast((v == '\\').to_bitmask()), // bs_bits - static_cast((v == '"').to_bitmask()), // quote_bits - }; -} - - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_STRINGPARSING_DEFS_H -/* end file simdjson/haswell/stringparsing_defs.h */ -/* end file simdjson/haswell/begin.h */ -/* including simdjson/generic/amalgamated.h for haswell: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for haswell */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for haswell: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for haswell */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for haswell */ -/* including simdjson/generic/jsoncharutils.h for haswell: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for haswell */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for haswell */ -/* including simdjson/generic/atomparsing.h for haswell: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for haswell */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace haswell { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for haswell */ -/* including simdjson/generic/dom_parser_implementation.h for haswell: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for haswell */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace haswell -} // namespace simdjson - -namespace simdjson { -namespace haswell { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for haswell */ -/* including simdjson/generic/implementation_simdjson_result_base.h for haswell: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for haswell */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for haswell */ -/* including simdjson/generic/numberparsing.h for haswell: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for haswell */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace haswell { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for haswell */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for haswell: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for haswell */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for haswell */ -/* end file simdjson/generic/amalgamated.h for haswell */ -/* including simdjson/haswell/end.h: #include "simdjson/haswell/end.h" */ -/* begin file simdjson/haswell/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_HASWELL -SIMDJSON_UNTARGET_REGION -#endif - -/* undefining SIMDJSON_IMPLEMENTATION from "haswell" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/haswell/end.h */ - -#endif // SIMDJSON_HASWELL_H -/* end file simdjson/haswell.h */ -/* including simdjson/haswell/implementation.h: #include */ -/* begin file simdjson/haswell/implementation.h */ -#ifndef SIMDJSON_HASWELL_IMPLEMENTATION_H -#define SIMDJSON_HASWELL_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_HASWELL -namespace simdjson { -namespace haswell { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation( - "haswell", - "Intel/AMD AVX2", - internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2 - ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_IMPLEMENTATION_H -/* end file simdjson/haswell/implementation.h */ - -/* including simdjson/haswell/begin.h: #include */ -/* begin file simdjson/haswell/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "haswell" */ -#define SIMDJSON_IMPLEMENTATION haswell - -/* including simdjson/haswell/base.h: #include "simdjson/haswell/base.h" */ -/* begin file simdjson/haswell/base.h */ -#ifndef SIMDJSON_HASWELL_BASE_H -#define SIMDJSON_HASWELL_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_HASWELL -namespace simdjson { -/** - * Implementation for Haswell (Intel AVX2). - */ -namespace haswell { - -class implementation; - -namespace { -namespace simd { -template struct simd8; -template struct simd8x64; -} // namespace simd -} // unnamed namespace - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_BASE_H -/* end file simdjson/haswell/base.h */ -/* including simdjson/haswell/intrinsics.h: #include "simdjson/haswell/intrinsics.h" */ -/* begin file simdjson/haswell/intrinsics.h */ -#ifndef SIMDJSON_HASWELL_INTRINSICS_H -#define SIMDJSON_HASWELL_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - * e.g., if __AVX2__ is set... in turn, we normally set these - * macros by compiling against the corresponding architecture - * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole - * software with these advanced instructions. In simdjson, we - * want to compile the whole program for a generic target, - * and only target our specific kernels. As a workaround, - * we directly include the needed headers. These headers would - * normally guard against such usage, but we carefully included - * (or ) before, so the headers - * are fooled. - */ -#include // for _blsr_u64 -#include // for __lzcnt64 -#include // for most things (AVX2, AVX512, _popcnt64) -#include -#include -#include -#include -#include // for _mm_clmulepi64_si128 -// unfortunately, we may not get _blsr_u64, but, thankfully, clang -// has it as a macro. -#ifndef _blsr_u64 -// we roll our own -#define _blsr_u64(n) ((n - 1) & n) -#endif // _blsr_u64 -#endif // SIMDJSON_CLANG_VISUAL_STUDIO - -static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for haswell kernel."); - -#endif // SIMDJSON_HASWELL_INTRINSICS_H -/* end file simdjson/haswell/intrinsics.h */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_HASWELL -// We enable bmi2 only if LLVM/clang is used, because GCC may not -// make good use of it. See https://github.com/simdjson/simdjson/pull/2243 -#if defined(__clang__) -SIMDJSON_TARGET_REGION("avx2,bmi,bmi2,pclmul,lzcnt,popcnt") -#else -SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt,popcnt") -#endif -#endif - -/* including simdjson/haswell/bitmanipulation.h: #include "simdjson/haswell/bitmanipulation.h" */ -/* begin file simdjson/haswell/bitmanipulation.h */ -#ifndef SIMDJSON_HASWELL_BITMANIPULATION_H -#define SIMDJSON_HASWELL_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmask.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return (int)_tzcnt_u64(input_num); -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - //////// - // You might expect the next line to be equivalent to - // return (int)_tzcnt_u64(input_num); - // but the generated code differs and might be less efficient? - //////// - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return _blsr_u64(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { - return int(_lzcnt_u64(input_num)); -} - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num);// Visual Studio wants two underscores -} -#else -simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); -} -#endif - - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_BITMANIPULATION_H -/* end file simdjson/haswell/bitmanipulation.h */ -/* including simdjson/haswell/bitmask.h: #include "simdjson/haswell/bitmask.h" */ -/* begin file simdjson/haswell/bitmask.h */ -#ifndef SIMDJSON_HASWELL_BITMASK_H -#define SIMDJSON_HASWELL_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { - // There should be no such thing with a processor supporting avx2 - // but not clmul. - __m128i all_ones = _mm_set1_epi8('\xFF'); - __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); - return _mm_cvtsi128_si64(result); -} - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_BITMASK_H -/* end file simdjson/haswell/bitmask.h */ -/* including simdjson/haswell/numberparsing_defs.h: #include "simdjson/haswell/numberparsing_defs.h" */ -/* begin file simdjson/haswell/numberparsing_defs.h */ -#ifndef SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H -#define SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace numberparsing { - -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - // this actually computes *16* values so we are being wasteful. - const __m128i ascii0 = _mm_set1_epi8('0'); - const __m128i mul_1_10 = - _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); - const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); - const __m128i mul_1_10000 = - _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); - const __m128i input = _mm_sub_epi8( - _mm_loadu_si128(reinterpret_cast(chars)), ascii0); - const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); - const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); - const __m128i t3 = _mm_packus_epi32(t2, t2); - const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); - return _mm_cvtsi128_si32( - t4); // only captures the sum of the first 8 digits, drop the rest -} - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace haswell -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H -/* end file simdjson/haswell/numberparsing_defs.h */ -/* including simdjson/haswell/simd.h: #include "simdjson/haswell/simd.h" */ -/* begin file simdjson/haswell/simd.h */ -#ifndef SIMDJSON_HASWELL_SIMD_H -#define SIMDJSON_HASWELL_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m256i value; - - // Zero constructor - simdjson_inline base() : value{__m256i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m256i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m256i&() const { return this->value; } - simdjson_inline operator __m256i&() { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm256_or_si256(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm256_and_si256(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8: base> { - typedef uint32_t bitmask_t; - typedef uint64_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m256i _value) : base>(_value) {} - - friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm256_cmpeq_epi8(lhs, rhs); } - - static const int SIZE = sizeof(base::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return _mm256_alignr_epi8(*this, _mm256_permute2x128_si256(prev_chunk, *this, 0x21), 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m256i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return _mm256_movemask_epi8(*this); } - simdjson_inline bool any() const { return !_mm256_testz_si256(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { return _mm256_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm256_setzero_si256(); } - static simdjson_inline simd8 load(const T values[32]) { - return _mm256_loadu_si256(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m256i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[32]) const { return _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), *this); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm256_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm256_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm256_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 32 - count_ones(mask) bytes of the result are significant but 32 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint32_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in four steps, first 8 bytes and then second 8 bytes... - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits - uint8_t mask3 = uint8_t(mask >> 16); // ... - uint8_t mask4 = uint8_t(mask >> 24); // ... - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - __m256i shufmask = _mm256_set_epi64x(thintable_epi8[mask4], thintable_epi8[mask3], - thintable_epi8[mask2], thintable_epi8[mask1]); - // we increment by 0x08 the second half of the mask and so forth - shufmask = - _mm256_add_epi8(shufmask, _mm256_set_epi32(0x18181818, 0x18181818, - 0x10101010, 0x10101010, 0x08080808, 0x08080808, 0, 0)); - // this is the version "nearly pruned" - __m256i pruned = _mm256_shuffle_epi8(*this, shufmask); - // we still need to put the pieces back together. - // we compute the popcount of the first words: - int pop1 = BitsSetTable256mul2[mask1]; - int pop3 = BitsSetTable256mul2[mask3]; - - // then load the corresponding mask - // could be done with _mm256_loadu2_m128i but many standard libraries omit this intrinsic. - __m256i v256 = _mm256_castsi128_si256( - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8))); - __m256i compactmask = _mm256_insertf128_si256(v256, - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop3 * 8)), 1); - __m256i almostthere = _mm256_shuffle_epi8(pruned, compactmask); - // We just need to write out the result. - // This is the tricky bit that is hard to do - // if we want to return a SIMD register, since there - // is no single-instruction approach to recombine - // the two 128-bit lanes with an offset. - __m128i v128; - v128 = _mm256_castsi256_si128(almostthere); - _mm_storeu_si128( reinterpret_cast<__m128i *>(output), v128); - v128 = _mm256_extractf128_si256(almostthere, 1); - _mm_storeu_si128( reinterpret_cast<__m128i *>(output + 16 - count_ones(mask & 0xFFFF)), v128); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, - int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, - int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31 - ) : simd8(_mm256_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v16,v17,v18,v19,v20,v21,v22,v23, - v24,v25,v26,v27,v28,v29,v30,v31 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epi8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return _mm256_cmpgt_epi8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm256_cmpgt_epi8(other, *this); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, - uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, - uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31 - ) : simd8(_mm256_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v16,v17,v18,v19,v20,v21,v22,v23, - v24,v25,v26,v27,v28,v29,v30,v31 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm256_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm256_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epu8(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7-N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 2, "Haswell kernel should use two registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+32)} {} - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - uint32_t mask1 = uint32_t(mask); - uint32_t mask2 = uint32_t(mask >> 32); - this->chunks[0].compress(mask1, output); - this->chunks[1].compress(mask2, output + 32 - count_ones(mask1)); - return 64 - count_ones(mask); - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r_hi = this->chunks[1].to_bitmask(); - return r_lo | (r_hi << 32); - } - - simdjson_inline simd8 reduce_or() const { - return this->chunks[0] | this->chunks[1]; - } - - simdjson_inline simd8x64 bit_or(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] | mask, - this->chunks[1] | mask - ); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_SIMD_H -/* end file simdjson/haswell/simd.h */ -/* including simdjson/haswell/stringparsing_defs.h: #include "simdjson/haswell/stringparsing_defs.h" */ -/* begin file simdjson/haswell/stringparsing_defs.h */ -#ifndef SIMDJSON_HASWELL_STRINGPARSING_DEFS_H -#define SIMDJSON_HASWELL_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 15 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - // store to dest unconditionally - we can overwrite the bits we don't like later - v.store(dst); - return { - static_cast((v == '\\').to_bitmask()), // bs_bits - static_cast((v == '"').to_bitmask()), // quote_bits - }; -} - - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_STRINGPARSING_DEFS_H -/* end file simdjson/haswell/stringparsing_defs.h */ -/* end file simdjson/haswell/begin.h */ -/* including generic/amalgamated.h for haswell: #include */ -/* begin file generic/amalgamated.h for haswell */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) -#error generic/dependencies.h must be included before generic/amalgamated.h! -#endif - -/* including generic/base.h for haswell: #include */ -/* begin file generic/base.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { - -struct json_character_block; - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_BASE_H -/* end file generic/base.h for haswell */ -/* including generic/dom_parser_implementation.h for haswell: #include */ -/* begin file generic/dom_parser_implementation.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// Interface a dom parser implementation must fulfill -namespace simdjson { -namespace haswell { -namespace { - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); -simdjson_inline bool is_ascii(const simd8x64& input); - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file generic/dom_parser_implementation.h for haswell */ -/* including generic/json_character_block.h for haswell: #include */ -/* begin file generic/json_character_block.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { - -struct json_character_block { - static simdjson_inline json_character_block classify(const simd::simd8x64& in); - - simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_inline uint64_t op() const noexcept { return _op; } - simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } - - uint64_t _whitespace; - uint64_t _op; -}; - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H -/* end file generic/json_character_block.h for haswell */ -/* end file generic/amalgamated.h for haswell */ -/* including generic/stage1/amalgamated.h for haswell: #include */ -/* begin file generic/stage1/amalgamated.h for haswell */ -// Stuff other things depend on -/* including generic/stage1/base.h for haswell: #include */ -/* begin file generic/stage1/base.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace stage1 { - -class bit_indexer; -template -struct buf_block_reader; -struct json_block; -class json_minifier; -class json_scanner; -struct json_string_block; -class json_string_scanner; -class json_structural_indexer; - -} // namespace stage1 - -namespace utf8_validation { -struct utf8_checker; -} // namespace utf8_validation - -using utf8_validation::utf8_checker; - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H -/* end file generic/stage1/base.h for haswell */ -/* including generic/stage1/buf_block_reader.h for haswell: #include */ -/* begin file generic/stage1/buf_block_reader.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace haswell { -namespace { -namespace stage1 { - -// Walks through a buffer in block-sized increments, loading the last part with spaces -template -struct buf_block_reader { -public: - simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t *full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t *dst) const; - /** - * Like get_remainder, for input followed by SIMDJSON_PADDING readable bytes. - * - * get_remainder writes the block with a memset and a memcpy of the remaining - * length, and the vector loads that read it back cannot be forwarded from - * those narrower stores: they wait until the stores reach the cache, which - * costs tens of cycles on every document. Here the block is put together in - * vector registers, from loads that may read into the padding, and written - * with full-width stores that the loads can forward from. - */ - simdjson_inline size_t get_remainder_padded(uint8_t *dst) const; - simdjson_inline void advance(); -private: - const uint8_t *buf; - const size_t len; - const size_t lenminusstep; - size_t idx; -}; - -template -simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} - -template -simdjson_inline size_t buf_block_reader::block_index() { return idx; } - -template -simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; -} - -template -simdjson_inline const uint8_t *buf_block_reader::full_block() const { - return &buf[idx]; -} - -template -simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { - if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; -} - -// keep_mask[128 - n + i] is 0xFF for i < n and 0 otherwise. -alignas(64) static const uint8_t keep_mask[256] = { - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF -}; - -template -simdjson_inline size_t buf_block_reader::get_remainder_padded(uint8_t *dst) const { - static_assert(STEP_SIZE <= 128, "keep_mask covers blocks of up to 128 bytes"); - static_assert(sizeof(simd::simd8) <= SIMDJSON_PADDING, "a vector load at the last byte must stay in the padding"); - const size_t remaining = len - idx; - if (remaining == 0) { return 0; } - const simd::simd8 spaces = simd::simd8::splat(0x20); - for (size_t off = 0; off < STEP_SIZE; off += sizeof(simd::simd8)) { - simd::simd8 v = spaces; - if (off < remaining) { - // Starts before len, so it ends within the padding. - const simd::simd8 in = simd::simd8::load(buf + idx + off); - const simd::simd8 keep = simd::simd8::load(keep_mask + 128 - remaining + off); - v = spaces ^ ((in ^ spaces) & keep); - } - v.store(dst + off); - } - return remaining; -} - -template -simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for haswell */ -/* including generic/stage1/json_escape_scanner.h for haswell: #include */ -/* begin file generic/stage1/json_escape_scanner.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace stage1 { - -/** - * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). - */ -struct json_escape_scanner { - /** The actual escape characters (the backslashes themselves). */ - uint64_t next_is_escaped = 0ULL; - - struct escaped_and_escape { - /** - * Mask of escaped characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 0100100010100101000 - * n \ \ n \ \ - * ``` - */ - uint64_t escaped; - /** - * Mask of escape characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 1001000101001010001 - * \ \ \ \ \ \ \ - * ``` - */ - uint64_t escape; - }; - - /** - * Get a mask of both escape and escaped characters (the characters following a backslash). - * - * @param potential_escape A mask of the character that can escape others (but could be - * escaped itself). e.g. block.eq('\\') - */ - simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { - -#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT - if (!backslash) { return {next_escaped_without_backslashes(), 0}; } -#endif - - // | | Mask (shows characters instead of 1's) | Depth | Instructions | - // |--------------------------------|----------------------------------------|-------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | - // | | ` even odd even odd odd` | | | - // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) - // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) - // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) - // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () - // (*) this is not needed until the next iteration - uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); - uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); - uint64_t escape = escape_and_terminal_code & backslash; - this->next_is_escaped = escape >> 63; - return {escaped, escape}; - } - -private: - static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; - - simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { - uint64_t escaped = this->next_is_escaped; - this->next_is_escaped = 0; - return escaped; - } - - /** - * Returns a mask of the next escape characters (masking out escaped backslashes), along with - * any non-backslash escape codes. - * - * \n \\n \\\n \\\\n returns: - * \n \ \ \n \ \ - * 11 100 1011 10100 - * - * You are expected to mask out the first bit yourself if the previous block had a trailing - * escape. - * - * & the result with potential_escape to get just the escape characters. - * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. - */ - static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { - // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: - // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be - // inverted (\\\ would be 010 instead of 101). - // - // ``` - // string: | ____\\\\_\\\\_____ | - // maybe_escaped | ODD | \ \ \ \ | - // even-aligned ^^^ ^^^^ odd-aligned - // ``` - // - // Taking that into account, our basic strategy is: - // - // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for - // odd-aligned runs. - // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the - // odd bits in odd-aligned runs. - // 3. & with backslash to clean up any stray bits. - // runs are set to 0, and then XORing with "odd": - // - // | | Mask (shows characters instead of 1's) | Instructions | - // |--------------------------------|----------------------------------------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | - // | | ` even odd even odd odd` | - // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) - // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) - // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) - // - - // Escaped characters are characters following an escape. - uint64_t maybe_escaped = potential_escape << 1; - - // To distinguish odd from even escape sequences, therefore, we turn on any *starting* - // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) - // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. - // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. - // - All other odd bytes are 1, and even bytes are 0. - uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; - uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; - - // Now we flip all odd bytes back with xor. This: - // - Makes odd runs of backslashes go from 0000 to 1010 - // - Makes even runs of backslashes go from 1111 to 1010 - // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) - // - Resets all other bytes to 0 - return even_series_codes_and_odd_bits ^ ODD_BITS; - } -}; - -} // namespace stage1 -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_escape_scanner.h for haswell */ -/* including generic/stage1/json_string_scanner.h for haswell: #include */ -/* begin file generic/stage1/json_string_scanner.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace stage1 { - -struct json_string_block { - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : - _escaped(escaped), _quote(quote), _in_string(in_string) {} - - // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } - // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } - // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } - // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } - - // escaped characters (backslashed--does not include the hex characters after \u) - uint64_t _escaped; - // real quotes (non-escaped ones) - uint64_t _quote; - // string characters (includes start quote but not end quote) - uint64_t _in_string; -}; - -// Scans blocks for string characters, storing the state necessary to do so -class json_string_scanner { -public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); - -private: - // Scans for escape characters - json_escape_scanner escape_scanner{}; - // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). - uint64_t prev_in_string = 0ULL; -}; - -// -// Return a mask of all string characters plus end quotes. -// -// prev_escaped is overflow saying whether the next character is escaped. -// prev_in_string is overflow saying whether we're still in a string. -// -// Backslash sequences outside of quotes will be detected in stage 2. -// -simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = escape_scanner.next(backslash).escaped; - const uint64_t quote = in.eq('"') & ~escaped; - - // - // prefix_xor flips on bits inside the string (and flips off the end quote). - // - // Then we xor with prev_in_string: if we were in a string already, its effect is flipped - // (characters inside strings are outside, and characters outside strings are inside). - // - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - - // - // Check if we're still in a string at the end of the box so the next block will know - // - prev_in_string = uint64_t(static_cast(in_string) >> 63); - - // Use ^ to turn the beginning quote off, and the end quote on. - - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_string_block(escaped, quote, in_string); -} - -simdjson_really_inline error_code json_string_scanner::finish() { - if (prev_in_string) { - return UNCLOSED_STRING; - } - return SUCCESS; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_string_scanner.h for haswell */ -/* including generic/stage1/utf8_lookup4_algorithm.h for haswell: #include */ -/* begin file generic/stage1/utf8_lookup4_algorithm.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace utf8_validation { - -using namespace simd; - - simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { -// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) -// Bit 1 = Too Long (ASCII followed by continuation) -// Bit 2 = Overlong 3-byte -// Bit 4 = Surrogate -// Bit 5 = Overlong 2-byte -// Bit 7 = Two Continuations - constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ - // 11______ 11______ - constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ - constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ - constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ - constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ - constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ - constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ - // 11110100 101_____ - // 11110101 1001____ - // 11110101 101_____ - // 1111011_ 1001____ - // 1111011_ 101_____ - // 11111___ 1001____ - // 11111___ 101_____ - constexpr const uint8_t TOO_LARGE_1000 = 1<<6; - // 11110101 1000____ - // 1111011_ 1000____ - // 11111___ 1000____ - constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ - - const simd8 byte_1_high = prev1.shr<4>().lookup_16( - // 0_______ ________ - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - // 10______ ________ - TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, - // 1100____ ________ - TOO_SHORT | OVERLONG_2, - // 1101____ ________ - TOO_SHORT, - // 1110____ ________ - TOO_SHORT | OVERLONG_3 | SURROGATE, - // 1111____ ________ - TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 - ); - constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( - // ____0000 ________ - CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, - // ____0001 ________ - CARRY | OVERLONG_2, - // ____001_ ________ - CARRY, - CARRY, - - // ____0100 ________ - CARRY | TOO_LARGE, - // ____0101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____011_ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - - // ____1___ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____1101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000 - ); - const simd8 byte_2_high = input.shr<4>().lookup_16( - // ________ 0_______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - - // ________ 1000____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, - // ________ 1001____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, - // ________ 101_____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - - // ________ 11______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT - ); - return (byte_1_high & byte_1_low & byte_2_high); - } - simdjson_inline simd8 check_multibyte_lengths(const simd8 input, - const simd8 prev_input, const simd8 sc) { - simd8 prev2 = input.prev<2>(prev_input); - simd8 prev3 = input.prev<3>(prev_input); - simd8 must23 = must_be_2_3_continuation(prev2, prev3); - simd8 must23_80 = must23 & uint8_t(0x80); - return must23_80 ^ sc; - } - - // - // Return nonzero if there are incomplete multibyte characters at the end of the block: - // e.g. if there is a 4-byte character, but it's 3 bytes from the end. - // - simdjson_inline simd8 is_incomplete(const simd8 input) { - // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): - // ... 1111____ 111_____ 11______ -#if SIMDJSON_IMPLEMENTATION_ICELAKE || (SIMDJSON_IMPLEMENTATION_RVV_VLS && __riscv_v_fixed_vlen >= 512) - static const uint8_t max_array[64] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#else - static const uint8_t max_array[32] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#endif - const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); - return input.gt_bits(max_value); - } - - struct utf8_checker { - // If this is nonzero, there has been a UTF-8 error. - simd8 error; - // The last input we received - simd8 prev_input_block; - // Whether the last input we received was incomplete (used for ASCII fast path) - simd8 prev_incomplete; - - // - // Check whether the current bytes are valid UTF-8. - // - simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { - // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes - // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) - simd8 prev1 = input.prev<1>(prev_input); - simd8 sc = check_special_cases(input, prev1); - this->error |= check_multibyte_lengths(input, prev_input, sc); - } - - // The only problem that can happen at EOF is that a multibyte character is too short - // or a byte value too large in the last bytes: check_special_cases only checks for bytes - // too large in the first of two bytes. - simdjson_inline void check_eof() { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; - } - - simdjson_inline void check_next_input(const simd8x64& input) { - if(simdjson_likely(is_ascii(input))) { - this->error |= this->prev_incomplete; - } else { - // you might think that a for-loop would work, but under Visual Studio, it is not good enough. - static_assert((simd8x64::NUM_CHUNKS == 1) - ||(simd8x64::NUM_CHUNKS == 2) - || (simd8x64::NUM_CHUNKS == 4), - "We support one, two or four chunks per 64-byte block."); - SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 1) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 2) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 4) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - this->check_utf8_bytes(input.get<2>(), input.get<1>()); - this->check_utf8_bytes(input.get<3>(), input.get<2>()); - } - this->prev_incomplete = is_incomplete(input.get::NUM_CHUNKS-1>()); - this->prev_input_block = input.get::NUM_CHUNKS-1>(); - } - } - // do not forget to call check_eof! - simdjson_warn_unused simdjson_inline error_code errors() { - return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; - } - - }; // struct utf8_checker -} // namespace utf8_validation - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H -/* end file generic/stage1/utf8_lookup4_algorithm.h for haswell */ -/* including generic/stage1/json_scanner.h for haswell: #include */ -/* begin file generic/stage1/json_scanner.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace stage1 { - -/** - * A block of scanned json, with information on operators and scalars. - * - * We seek to identify pseudo-structural characters. Anything that is inside - * a string must be omitted (hence & ~_string.string_tail()). - * Otherwise, pseudo-structural characters come in two forms. - * 1. We have the structural characters ([,],{,},:, comma). The - * term 'structural character' is from the JSON RFC. - * 2. We have the 'scalar pseudo-structural characters'. - * Scalars are quotes, and any character except structural characters and white space. - * - * To identify the scalar pseudo-structural characters, we must look at what comes - * before them: it must be a space, a quote or a structural characters. - * Starting with simdjson v0.3, we identify them by - * negation: we identify everything that is followed by a non-quote scalar, - * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. - */ -struct json_block { -public: - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - - /** - * The start of structurals. - * In simdjson prior to v0.3, these were called the pseudo-structural characters. - **/ - simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } - /** All JSON whitespace (i.e. not in a string) */ - simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } - - // Helpers - - /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } - /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } - - // string and escape characters - json_string_block _string; - // whitespace, structural characters ('operators'), scalars - json_character_block _characters; - // whether the previous character was a scalar - uint64_t _follows_potential_nonquote_scalar; -private: - // Potential structurals (i.e. disregarding strings) - - /** - * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". - * They may reside inside a string. - **/ - simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } - /** - * The start of non-operator runs, like 123, true and "abc". - * It main reside inside a string. - **/ - simdjson_inline uint64_t potential_scalar_start() const noexcept { - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space - // then we know that it is irrelevant structurally. - return _characters.scalar() & ~follows_potential_scalar(); - } - /** - * Whether the given character is immediately after a non-operator like 123, true. - * The characters following a quote are not included. - */ - simdjson_inline uint64_t follows_potential_scalar() const noexcept { - // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character - // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a - // white space. - // It is understood that within quoted region, anything at all could be marked (irrelevant). - return _follows_potential_nonquote_scalar; - } -}; - -/** - * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. - * - * The scanner starts by calculating two distinct things: - * - string characters (taking \" into account) - * - structural characters or 'operators' ([]{},:, comma) - * and scalars (runs of non-operators like 123, true and "abc") - * - * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: - * in particular, the operator/scalar bit will find plenty of things that are actually part of - * strings. When we're done, json_block will fuse the two together by masking out tokens that are - * part of a string. - */ -class json_scanner { -public: - json_scanner() = default; - simdjson_inline json_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_warn_unused simdjson_inline error_code finish(); - -private: - // Whether the last character of the previous iteration is part of a scalar token - // (anything except whitespace or a structural character/'operator'). - uint64_t prev_scalar = 0ULL; - json_string_scanner string_scanner{}; -}; - - -// -// Check if the current character immediately follows a matching character. -// -// For example, this checks for quotes with backslashes in front of them: -// -// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); -// -simdjson_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; -} - -simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { - json_string_block strings = string_scanner.next(in); - // identifies the white-space and the structural characters - json_character_block characters = json_character_block::classify(in); - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). - // - // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) - // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential - // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we - // may need to add an extra check when parsing strings. - // - // Performance: there are many ways to skin this cat. - const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); - uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_block( - strings,// strings is a function-local object so either it moves or the copy is elided. - characters, - follows_nonquote_scalar - ); -} - -simdjson_warn_unused simdjson_inline error_code json_scanner::finish() { - return string_scanner.finish(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H -/* end file generic/stage1/json_scanner.h for haswell */ - -// All other declarations -/* including generic/stage1/find_next_document_index.h for haswell: #include */ -/* begin file generic/stage1/find_next_document_index.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace stage1 { - -/** - * This algorithm is used to quickly identify the last structural position that - * makes up a complete document. - * - * It does this by going backwards and finding the last *document boundary* (a - * place where one value follows another without a comma between them). If the - * last document (the characters after the boundary) has an equal number of - * start and end brackets, it is considered complete. - * - * Simply put, we iterate over the structural characters, starting from - * the end. We consider that we found the end of a JSON document when the - * first element of the pair is NOT one of these characters: '{' '[' ':' ',' - * and when the second element is NOT one of these characters: '}' ']' ':' ','. - * - * This simple comparison works most of the time, but it does not cover cases - * where the batch's structural indexes contain a perfect amount of documents. - * In such a case, we do not have access to the structural index which follows - * the last document, therefore, we do not have access to the second element in - * the pair, and that means we cannot identify the last document. To fix this - * issue, we keep a count of the open and closed curly/square braces we found - * while searching for the pair. When we find a pair AND the count of open and - * closed curly/square braces is the same, we know that we just passed a - * complete document, therefore the last json buffer location is the end of the - * batch. - */ -simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { - const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - const uint8_t e = parser.buf[len - 1]; - return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && - e != ' ' && e != '\t' && e != '\n' && e != '\r'; -} - -/** - * The structural index past the last one of a partial batch is the end of this - * window, not a token of it: the byte there is the first byte of the next - * window. A finished value must not be rejected because that byte is ',', ':', - * ']' or '}'. A last document that itself ends with '{', '[', ',' or ':' is - * invalid JSON, yet its brackets may balance: walking it, the On-Demand API - * could read the next batch as a value. We leave such a document to the next - * batch, so that the batch ends between two documents. - */ -simdjson_inline bool ends_with_unfinished_document(dom_parser_implementation &parser) { - const uint8_t last = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - return last == '{' || last == '[' || last == ',' || last == ':'; -} - -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { - // Variant: do not count separately, just figure out depth - if(parser.n_structural_indexes == 0) { return 0; } - auto arr_cnt = 0; - auto obj_cnt = 0; - for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { - auto idxb = parser.structural_indexes[i]; - switch (parser.buf[idxb]) { - case ':': - case ',': - continue; - case '}': - obj_cnt--; - continue; - case ']': - arr_cnt--; - continue; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - auto idxa = parser.structural_indexes[i - 1]; - switch (parser.buf[idxa]) { - case '{': - case '[': - case ':': - case ',': - continue; - } - // Last document is complete, so the next document will appear after! - if (!arr_cnt && !obj_cnt) { - return defer_last ? i : parser.n_structural_indexes; - } - // Last document is incomplete; mark the document at i + 1 as the next one - return i; - } - // If we made it to the end, we want to finish counting to see if we have a full document. - switch (parser.buf[parser.structural_indexes[0]]) { - case '}': - obj_cnt--; - break; - case ']': - arr_cnt--; - break; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - if (!arr_cnt && !obj_cnt) { - // We have a complete document. - return defer_last ? 0 : parser.n_structural_indexes; - } - return 0; -} - -/** - * Sentinel value returned to indicate a document started but didn't fit - * (CAPACITY error), as opposed to 0 which means no document content found - * (EMPTY). - */ -constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; - -/** - * For RFC 7464 JSON text sequences, filter RS from structural indexes and - * find batch boundaries. - * - * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. - * RS bytes appear in structural_indexes as they are classified as scalars. - * This function: - * 1. Scans structural_indexes to find and count RS positions - * 2. Filters RS out of structural_indexes in-place - * 3. Determines batch boundaries based on RS positions - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @param scan_len Offset past which no value start may be derived. Equals len - * unless stage 1 dropped a trailing unclosed string, whose bytes it - * never validated. - * @return The number of structural indexes to keep (after RS filtering), - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t find_next_document_index_json_sequence( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start, - size_t scan_len) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Phase 1: Scan structural_indexes to find RS positions and handle them. - // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS - // is already in structural_indexes (it's an operator). For scalars like numbers, - // the digit following RS is NOT in structural_indexes because the scanner sees - // RS as a scalar, making the digit a scalar continuation, not a start. - // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the - // actual value start position. - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_rs_pos = 0; - uint32_t rs_count = 0; - - for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { - const uint32_t pos = parser.structural_indexes[read_idx]; - if (parser.buf[pos] == 0x1E) { - // This is an RS character - find the actual JSON value start. - last_rs_pos = pos; - rs_count++; - // Skip past this RS and any whitespace *and any additional RSes* - // to locate the real value. Consecutive RSes are degenerate - // "empty records" per RFC 7464; we collapse them here. They do - // not always appear as separate entries in structural_indexes - // because the scanner groups runs of adjacent non-whitespace - // scalar bytes (including RS) into a single scalar start. - uint32_t value_start = pos + 1; - while (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { - value_start++; - } else if (c == 0x1E) { - // Collapsed empty record. Still count it so rs_count reflects - // the true number of record markers and last_rs_pos tracks - // the final one. - last_rs_pos = value_start; - rs_count++; - value_start++; - } else { - break; - } - } - // If the scanner emitted additional structurals inside the - // whitespace+RS run we just walked over (i.e., isolated RSes - // separated by whitespace), skip past them so we do not - // double-count or double-emit. - while (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] < value_start) { - read_idx++; - } - // Check if the value start is an operator (always present in - // scanner structural_indexes) or a scalar-like start (which may - // be missing from structural_indexes and must be added here). - // Note: '"' is NOT always in structural_indexes. The scanner - // classifies '"' as a scalar character and emits it as a - // structural only when it is a *scalar start* (preceded by - // whitespace or an operator). When '"' immediately follows an - // RS (which the scanner also classifies as scalar), it is - // treated as a scalar continuation and not emitted - so we - // must add it here just like any other scalar value. - if (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - const bool is_operator = - (c == '{' || c == '}' || c == '[' || c == ']' || - c == ':' || c == ','); - // If the next scanner structural is exactly at value_start, - // the scanner already emitted it (it followed whitespace) and - // we must not add a duplicate - a subsequent iteration will - // copy it into write_idx. - const bool already_emitted = - (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] == value_start); - if (!is_operator && !already_emitted) { - // Scalar value (number/true/false/null/string) - add its - // position since scanner missed it. - parser.structural_indexes[write_idx++] = value_start; - } - } - } else { - // Not RS, copy to output - parser.structural_indexes[write_idx++] = pos; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { - // Only RS markers here: the last one opens a record continuing past the - // window, so that is where the next batch resumes. - if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } - return 0; - } - if (rs_count == 0) { - // No RS found; for final batch, try generic boundary detection - return is_final ? find_next_document_index(parser) : 0; - } - - // Phase 2: Determine batch boundaries based on RS positions - - if (is_final) { - // Final batch: all documents are complete (last one ends at EOF). - // In json_sequence mode, RS markers define document boundaries, so all - // remaining structurals form complete documents. Return them all directly. - // (Calling find_next_document_index() would fail for scalar documents.) - return parser.n_structural_indexes; - } - - // Partial batch: need to find complete documents only. - // A document starting at an RS is complete if there is another RS after it. - next_batch_start = last_rs_pos; - - if (rs_count < 2) { - // Only one RS, so we have at most one document that may be incomplete. - // We cannot confirm it is complete without another RS. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least 2 RS markers. The last complete document ends before last_rs_pos. - - // Find the structural index cutoff: keep only structurals < last_rs_pos. - // Since we already filtered RS, all remaining structurals are valid. - // We iterate backward to find the last structural before last_rs_pos. - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_rs_pos) { - keep_count = i; - break; - } - } - - // No structurals before the last RS - no complete documents - if (keep_count == 0) { return 0; } - - // All documents before the last RS are complete by definition (the next RS - // confirms their end). No need to call find_next_document_index() which - // would fail for scalar documents like `1` or `"hello"`. - return keep_count; -} - -/** - * Filter comma-delimited documents by removing root-level commas from - * structural indexes. - * - * For comma-delimited format like `{...},{...},{...}`, we need to remove - * the commas that separate documents (depth 0) while preserving commas - * inside arrays and objects (depth > 0). - * - * After filtering, the structural indexes look like whitespace-delimited - * documents, so find_next_document_index() works unchanged. - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @return The number of structural indexes to keep, - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t filter_comma_delimited( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Track depth to identify root-level commas (depth 0) - int depth = 0; - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_root_comma_pos = 0; - uint32_t root_comma_count = 0; - - for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { - uint32_t idx = parser.structural_indexes[i]; - uint8_t c = parser.buf[idx]; - - switch (c) { - case '{': case '[': - depth++; - parser.structural_indexes[write_idx++] = idx; - break; - case '}': case ']': - depth--; - parser.structural_indexes[write_idx++] = idx; - break; - case ',': - if (depth == 0) { - // Root-level comma = document boundary, skip it - last_root_comma_pos = idx; - root_comma_count++; - continue; - } - parser.structural_indexes[write_idx++] = idx; - break; - default: - // Colons, scalars, etc. - parser.structural_indexes[write_idx++] = idx; - break; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { return 0; } - - if (is_final) { - // Final batch: use standard boundary detection on filtered indexes - return find_next_document_index(parser); - } - - // Partial batch: need to find complete documents only. - // A document ending with a root comma is complete. - if (root_comma_count == 0) { - // No root commas found; we cannot confirm any document is complete. - // The whole batch might be one incomplete document. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least one root comma. Documents before the last comma are complete. - next_batch_start = last_root_comma_pos + 1; - - // Find the structural index cutoff: keep only structurals < last_root_comma_pos - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_root_comma_pos) { - keep_count = i; - break; - } - } - - if (keep_count == 0) { return 0; } - - // Use standard boundary detection on the complete portion - parser.n_structural_indexes = keep_count; - return find_next_document_index(parser); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H -/* end file generic/stage1/find_next_document_index.h for haswell */ -/* including generic/stage1/json_minifier.h for haswell: #include */ -/* begin file generic/stage1/json_minifier.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace haswell { -namespace { -namespace stage1 { - -class json_minifier { -public: - template - static error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept; - -private: - simdjson_inline json_minifier(uint8_t *_dst) - : dst{_dst} - {} - template - simdjson_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_warn_unused simdjson_inline error_code finish(uint8_t *dst_start, size_t &dst_len); - json_scanner scanner{}; - uint8_t *dst; -}; - -simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { - uint64_t mask = block.whitespace(); - dst += in.compress(mask, dst); -} - -simdjson_warn_unused simdjson_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { - error_code error = scanner.finish(); - if (error) { dst_len = 0; return error; } - dst_len = dst - dst_start; - return SUCCESS; -} - -template<> -simdjson_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - simd::simd8x64 in_2(block_buf+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1); - this->next(in_2, block_2); - reader.advance(); -} - -template<> -simdjson_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - json_block block_1 = scanner.next(in_1); - this->next(block_buf, block_1); - reader.advance(); -} - -template -error_code json_minifier::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept { - buf_block_reader reader(buf, len); - json_minifier minifier(dst); - - // Index the first n-1 blocks - while (reader.has_full_block()) { - minifier.step(reader.full_block(), reader); - } - - // Index the last (remainder) block, padded with spaces - uint8_t block[STEP_SIZE]; - size_t remaining_bytes = reader.get_remainder(block); - if (remaining_bytes > 0) { - // We do not want to write directly to the output stream. Rather, we write - // to a local buffer (for safety). - uint8_t out_block[STEP_SIZE]; - uint8_t * const guarded_dst{minifier.dst}; - minifier.dst = out_block; - minifier.step(block, reader); - size_t to_write = minifier.dst - out_block; - // In some cases, we could be enticed to consider the padded spaces - // as part of the string. This is fine as long as we do not write more - // than we consumed. - if(to_write > remaining_bytes) { to_write = remaining_bytes; } - memcpy(guarded_dst, out_block, to_write); - minifier.dst = guarded_dst + to_write; - } - return minifier.finish(dst, dst_len); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H -/* end file generic/stage1/json_minifier.h for haswell */ -/* including generic/stage1/json_structural_indexer.h for haswell: #include */ -/* begin file generic/stage1/json_structural_indexer.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace haswell { -namespace { -namespace stage1 { - -class bit_indexer { -public: - uint32_t *tail; - - simdjson_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} - -#if SIMDJSON_PREFER_REVERSE_BITS - /** - * ARM lacks a fast trailing zero instruction, but it has a fast - * bit reversal instruction and a fast leading zero instruction. - * Thus it may be profitable to reverse the bits (once) and then - * to rely on a sequence of instructions that call the leading - * zero instruction. - * - * Performance notes: - * The chosen routine is not optimal in terms of data dependency - * since zero_leading_bit might require two instructions. However, - * it tends to minimize the total number of instructions which is - * beneficial. - */ - simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { - int lz = leading_zeroes(rev_bits); - this->tail[i] = static_cast(idx) + lz; - rev_bits = zero_leading_bit(rev_bits, lz); - } -#else - /** - * Under recent x64 systems, we often have both a fast trailing zero - * instruction and a fast 'clear-lower-bit' instruction so the following - * algorithm can be competitive. - */ - - simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { - this->tail[i] = idx + trailing_zeroes(bits); - bits = clear_lowest_bit(bits); - } -#endif // SIMDJSON_PREFER_REVERSE_BITS - - template - simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { - write_index(idx, bits, START); - SIMDJSON_IF_CONSTEXPR (N > 1) { - write_indexes<(N-1>0?START+1:START), (N-1>=0?N-1:1)>(idx, bits); - } - return START+N; - } - - template - simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { - write_indexes(idx, bits); - SIMDJSON_IF_CONSTEXPR ((START+STEP) < END) { - if (simdjson_unlikely((START+STEP) < cnt)) { - write_indexes_stepped<(START+STEP(idx, bits, cnt); - } - } - return ((END-START) % STEP) == 0 ? END : (END-START) - ((END-START) % STEP) + STEP; - } - - // flatten out values in 'bits' assuming that they are are to have values of idx - // plus their position in the bitvector, and store these indexes at - // base_ptr[base] incrementing base as we go - // will potentially store extra values beyond end of valid bits, so base_ptr - // needs to be large enough to handle this - // - // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it - // will provide its own version of the code. -#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - simdjson_inline void write(uint32_t idx, uint64_t bits); -#else - simdjson_inline void write(uint32_t idx, uint64_t bits) { - // In some instances, the next branch is expensive because it is mispredicted. - // Unfortunately, in other cases, - // it helps tremendously. - if (bits == 0) - return; - - int cnt = static_cast(count_ones(bits)); - -#if SIMDJSON_PREFER_REVERSE_BITS - bits = reverse_bits(bits); -#endif -#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP - static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; -#else - static constexpr const int STEP = 4; -#endif - static constexpr const int STEP_UNTIL = 24; - - write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); - SIMDJSON_IF_CONSTEXPR (STEP_UNTIL < 64) { - if (simdjson_unlikely(STEP_UNTIL < cnt)) { - for (int i=STEP_UNTIL; itail += cnt; - } -#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -}; - -class json_structural_indexer { -public: - /** - * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. - * - * @param partial Setting the partial parameter to true allows the find_structural_bits to - * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If - * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. - */ - template - static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept; - -private: - simdjson_inline json_structural_indexer(uint32_t *structural_indexes); - template - simdjson_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_warn_unused simdjson_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); - - json_scanner scanner{}; - utf8_checker checker{}; - bit_indexer indexer; - uint64_t prev_structurals = 0; - uint64_t unescaped_chars_error = 0; -}; - -simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} - -// Skip the last character if it is partial -simdjson_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { - if (simdjson_unlikely(len < 3)) { - switch (len) { - case 2: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left - return len; - case 1: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - return len; - case 0: - return len; - } - } - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left - return len; -} - -// -// PERF NOTES: -// We pipe 2 inputs through these stages: -// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load -// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. -// 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. -// The output of step 1 depends entirely on this information. These functions don't quite use -// up enough CPU: the second half of the functions is highly serial, only using 1 execution core -// at a time. The second input's scans has some dependency on the first ones finishing it, but -// they can make a lot of progress before they need that information. -// 3. Step 1 does not use enough capacity, so we run some extra stuff while we're waiting for that -// to finish: utf-8 checks and generating the output from the last iteration. -// -// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all -// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough -// workout. -// -template -error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept { - if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } - // We guard the rest of the code so that we can assume that len > 0 throughout. - if (len == 0) { return EMPTY; } - if (is_streaming(partial)) { - len = trim_partial_utf8(buf, len); - // If you end up with an empty window after trimming - // the partial UTF-8 bytes, then chances are good that you - // have an UTF-8 formatting error. - if(len == 0) { return UTF8_ERROR; } - } - buf_block_reader reader(buf, len); - json_structural_indexer indexer(parser.structural_indexes.get()); - - // Read all but the last block - while (reader.has_full_block()) { - indexer.step(reader.full_block(), reader); - } - // Take care of the last block (will always be there unless file is empty which is - // not supposed to happen.) - uint8_t block[STEP_SIZE]; - const size_t remaining = parser._unpadded ? reader.get_remainder(block) : reader.get_remainder_padded(block); - if (simdjson_unlikely(remaining == 0)) { return UNEXPECTED_ERROR; } - indexer.step(block, reader); - return indexer.finish(parser, reader.block_index(), len, partial); -} - -template<> -simdjson_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block); - simd::simd8x64 in_2(block+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1, reader.block_index()); - this->next(in_2, block_2, reader.block_index()+64); - reader.advance(); -} - -template<> -simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block); - json_block block_1 = scanner.next(in_1); - this->next(in_1, block_1, reader.block_index()); - reader.advance(); -} - -simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { - uint64_t unescaped = in.lteq(0x1F); -#if SIMDJSON_UTF8VALIDATION - checker.check_next_input(in); -#endif - indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser - prev_structurals = block.structural_start(); - unescaped_chars_error |= block.non_quote_inside_string(unescaped); -} - -// Drops the unclosed string that ends a json_sequence batch and leaves its -// start in structural_indexes[n_structural_indexes]. The last index is its -// opening quote, unless the quote directly follows an RS: the scanner reads -// RS as a scalar byte, so it indexes the RS and not the quote. Keep the RS. -simdjson_inline void drop_json_sequence_unclosed_string(dom_parser_implementation &parser, size_t len) noexcept { - uint32_t quote = parser.structural_indexes[parser.n_structural_indexes - 1]; - if (parser.buf[quote] == '"') { - parser.n_structural_indexes--; - return; - } - while (quote < len && parser.buf[quote] != '"') { quote++; } - parser.structural_indexes[parser.n_structural_indexes] = quote; -} - -simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial) { - // Write out the final iteration's structurals - indexer.write(uint32_t(idx-64), prev_structurals); - error_code error = scanner.finish(); - // We deliberately break down the next expression so that it is - // human readable. - const bool should_we_exit = is_streaming(partial) ? - ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING - : (error != SUCCESS); // if partial is false, we must have SUCCESS - const bool have_unclosed_string = (error == UNCLOSED_STRING); - parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); - /*** - * The On-Demand API requires special padding. - * - * This is related to https://github.com/simdjson/simdjson/issues/906 - * Basically, we want to make sure that if the parsing continues beyond the last (valid) - * structural character, it quickly stops. - * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. - * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing - * continues, then it must be [,] or }. - * Suppose it is ] or }. We backtrack to the first character, what could it be that would - * not trigger an error? It could be ] or } but no, because you can't start a document that way. - * It can't be a comma, a colon or any simple value. So the only way we could continue is - * if the repeated character is [. But if so, the document must start with [. But if the document - * starts with [, it should end with ]. If we enforce that rule, then we would get - * ][[ which is invalid. - * - * This is illustrated with the test array_iterate_unclosed_error() on the following input: - * R"({ "a": [,,)" - **/ - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final - parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); - parser.structural_indexes[parser.n_structural_indexes + 2] = 0; - parser.next_structural_index = 0; - - // Bail out only once the count and sentinels above are set: - // document_stream::truncated_bytes() reads them even on error. - if (simdjson_unlikely(should_we_exit)) { return error; } - if (unescaped_chars_error) { return UNESCAPED_CHARS; } - - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - return EMPTY; - } - if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { - return UNEXPECTED_ERROR; - } - if (partial == stage1_mode::streaming_partial) { - // If we have an unclosed string, then the last structural - // will be the quote and we want to make sure to omit it. - if(have_unclosed_string) { - parser.n_structural_indexes--; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - // We truncate the input to the end of the last complete document (or zero). - const bool partial_scalar = !have_unclosed_string && ends_with_partial_scalar(parser, len); - const bool unfinished = ends_with_unfinished_document(parser); - auto new_structural_indexes = find_next_document_index(parser, partial_scalar || unfinished); - if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { - // The batch holds a single document, whose brackets balance but which - // cannot be valid JSON: it is not too big, it is invalid. - if (unfinished && find_next_document_index(parser, partial_scalar) != 0) { return TAPE_ERROR; } - if(parser.structural_indexes[0] == 0) { - // If the buffer is partial and we started at index 0 but the document is - // incomplete, it's too big to parse. - return CAPACITY; - } else { - // It is possible that the document could be parsed, we just had a lot - // of white space. - parser.n_structural_indexes = 0; - return EMPTY; - } - } - parser.n_structural_indexes = new_structural_indexes; - } else if (partial == stage1_mode::streaming_final) { - if(have_unclosed_string) { parser.n_structural_indexes--; } - // We truncate the input to the end of the last complete document (or zero). - // Because partial == stage1_mode::streaming_final, it means that we may - // silently ignore trailing garbage. Though it sounds bad, we do it - // deliberately because many people who have streams of JSON documents - // will truncate them for processing. E.g., imagine that you are uncompressing - // the data from a size file or receiving it in chunks from the network. You - // may not know where exactly the last document will be. Meanwhile the - // document_stream instances allow people to know the JSON documents they are - // parsing (see the iterator.source() method). - parser.n_structural_indexes = find_next_document_index(parser); - // We store the initial n_structural_indexes so that the client can see - // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, - // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, - // otherwise, it will copy some prior index. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - // This next line is critical, do not change it unless you understand what you are - // doing. - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - // We tolerate an unclosed string at the very end of the stream. Indeed, users - // often load their data in bulk without being careful and they want us to ignore - // the trailing garbage. - return EMPTY; - } - } else if (partial == stage1_mode::json_sequence_partial) { - // RFC 7464: use RS positions for batch boundaries - // A discarded unclosed string also caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - // The next batch starts at an RS, which the On-Demand API never reads as a - // string, a container or a separator. But an invalid last document may make - // it skip that RS as a value: repeat it, so that the index past it is not - // the next record, which can be a string that does not close in this batch. - parser.structural_indexes[parser.n_structural_indexes + 1] = next_batch_start; - } else if (partial == stage1_mode::json_sequence_final) { - // RFC 7464: final batch, last document extends to EOF - // As above: a discarded unclosed string caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); - // The filter compacted structural_indexes in place and restored the EOF - // sentinel past the compacted end, so the copy below is either the start - // of a truncated document or len, as in streaming_final. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } else if (partial == stage1_mode::comma_delimited_partial) { - // Comma-delimited: filter root-level commas, use comma positions for batch boundaries - if(have_unclosed_string) { - parser.n_structural_indexes--; - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - } else if (partial == stage1_mode::comma_delimited_final) { - // Comma-delimited: final batch, last document extends to EOF - if(have_unclosed_string) { parser.n_structural_indexes--; } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } - checker.check_eof(); - return checker.errors(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. -#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H -/* end file generic/stage1/json_structural_indexer.h for haswell */ -/* including generic/stage1/utf8_validator.h for haswell: #include */ -/* begin file generic/stage1/utf8_validator.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace stage1 { - -/** - * Validates that the string is actual UTF-8. - */ -template -bool generic_validate_utf8(const uint8_t * input, size_t length) { - checker c{}; - buf_block_reader<64> reader(input, length); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - c.check_next_input(in); - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - c.check_next_input(in); - reader.advance(); - c.check_eof(); - return c.errors() == error_code::SUCCESS; -} - -bool generic_validate_utf8(const char * input, size_t length) { - return generic_validate_utf8(reinterpret_cast(input),length); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H -/* end file generic/stage1/utf8_validator.h for haswell */ -/* end file generic/stage1/amalgamated.h for haswell */ -/* including generic/stage2/amalgamated.h for haswell: #include */ -/* begin file generic/stage2/amalgamated.h for haswell */ -// Stuff other things depend on -/* including generic/stage2/base.h for haswell: #include */ -/* begin file generic/stage2/base.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace stage2 { - -class json_iterator; -struct tape_builder; -struct tape_writer; - -} // namespace stage2 -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H -/* end file generic/stage2/base.h for haswell */ -/* including generic/stage2/tape_writer.h for haswell: #include */ -/* begin file generic/stage2/tape_writer.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace haswell { -namespace { -namespace stage2 { - -struct tape_writer { - /** The next place to write to tape */ - uint64_t *next_tape_loc; - - /** Write a signed 64-bit value to tape. */ - simdjson_inline void append_s64(int64_t value) noexcept; - - /** Write an unsigned 64-bit value to tape. */ - simdjson_inline void append_u64(uint64_t value) noexcept; - - /** Write a double value to tape. */ - simdjson_inline void append_double(double value) noexcept; - - /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ - simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; - - /** - * Append a tape entry (an 8-bit type,and 56 bits worth of value). - */ - simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; - - /** - * Skip the current tape entry without writing. - * - * Used to skip the start of the container, since we'll come back later to fill it in when the - * container ends. - */ - simdjson_inline void skip() noexcept; - - /** - * Skip the number of tape entries necessary to write a large u64 or i64. - */ - simdjson_inline void skip_large_integer() noexcept; - - /** - * Skip the number of tape entries necessary to write a double. - */ - simdjson_inline void skip_double() noexcept; - - /** - * Write a value to a known location on tape. - * - * Used to go back and write out the start of a container after the container ends. - */ - simdjson_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; - -private: - /** - * Append both the tape entry, and a supplementary value following it. Used for types that need - * all 64 bits, such as double and uint64_t. - */ - template - simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; -}; // struct tape_writer - -simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { - append2(0, value, internal::tape_type::INT64); -} - -simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { - append(0, internal::tape_type::UINT64); - *next_tape_loc = value; - next_tape_loc++; -} - -/** Write a double value to tape. */ -simdjson_inline void tape_writer::append_double(double value) noexcept { - append2(0, value, internal::tape_type::DOUBLE); -} - -simdjson_inline void tape_writer::skip() noexcept { - next_tape_loc++; -} - -simdjson_inline void tape_writer::skip_large_integer() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::skip_double() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { - *next_tape_loc = val | ((uint64_t(char(t))) << 56); - next_tape_loc++; -} - -template -simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { - append(val, t); - static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); - memcpy(next_tape_loc, &val2, sizeof(val2)); - next_tape_loc++; -} - -simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { - tape_loc = val | ((uint64_t(char(t))) << 56); -} - -simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { - // Write to string buffer: [4-byte LE length][digits][null] - uint32_t str_len = uint32_t(len); - memcpy(string_buf, &str_len, sizeof(uint32_t)); - memcpy(string_buf + sizeof(uint32_t), src, len); - string_buf[sizeof(uint32_t) + len] = 0; - // Tape entry: offset into string buffer - // The caller must set the offset relative to doc.string_buf base - append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches - string_buf += sizeof(uint32_t) + len + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H -/* end file generic/stage2/tape_writer.h for haswell */ -/* including generic/stage2/logger.h for haswell: #include */ -/* begin file generic/stage2/logger.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - - -// This is for an internal-only stage 2 specific logger. -// Set LOG_ENABLED = true to log what stage 2 is doing! -namespace simdjson { -namespace haswell { -namespace { -namespace logger { - - static constexpr const char * DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; - -#if SIMDJSON_VERBOSE_LOGGING - static constexpr const bool LOG_ENABLED = true; -#else - static constexpr const bool LOG_ENABLED = false; -#endif - static constexpr const int LOG_EVENT_LEN = 20; - static constexpr const int LOG_BUFFER_LEN = 30; - static constexpr const int LOG_SMALL_BUFFER_LEN = 10; - static constexpr const int LOG_INDEX_LEN = 5; - - static int log_depth; // Not threadsafe. Log only. - - // Helper to turn unprintable or newline characters into spaces - static simdjson_inline char printable_char(char c) { - if (c >= 0x20) { - return c; - } else { - return ' '; - } - } - - // Print the header and set up log_start - static simdjson_inline void log_start() { - if (LOG_ENABLED) { - log_depth = 0; - printf("\n"); - printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); - printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN+2, DASHES, LOG_BUFFER_LEN+2, DASHES, LOG_SMALL_BUFFER_LEN+2, DASHES, 5+2, DASHES); - } - } - - simdjson_unused static simdjson_inline void log_string(const char *message) { - if (LOG_ENABLED) { - printf("%s\n", message); - } - } - - // Logs a single line from the stage 2 DOM parser - template - static simdjson_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { - if (LOG_ENABLED) { - printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); - auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; - auto next_index = structurals.next_structural; - auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); - auto next = &structurals.buf[*next_index]; - { - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i=0;i */ -/* begin file generic/stage2/json_iterator.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace stage2 { - -class json_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - uint32_t depth{0}; - - /** - * Walk the JSON document. - * - * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as - * the first parameter; some callbacks have other parameters as well: - * - * - visit_document_start() - at the beginning. - * - visit_document_end() - at the end (if things were successful). - * - * - visit_array_start() - at the start `[` of a non-empty array. - * - visit_array_end() - at the end `]` of a non-empty array. - * - visit_empty_array() - when an empty array is encountered. - * - * - visit_object_end() - at the start `]` of a non-empty object. - * - visit_object_start() - at the end `]` of a non-empty object. - * - visit_empty_object() - when an empty object is encountered. - * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is - * guaranteed to point at the first quote of the string (`"key"`). - * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. - * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. - * - * - increment_count(iter) - each time a value is found in an array or object. - */ - template - simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; - - /** - * Create an iterator capable of walking a JSON document. - * - * The document must have already passed through stage 1. - */ - simdjson_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); - - /** - * Look at the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *peek(const uint32_t *batch_end = nullptr) const noexcept; - /** - * Advance to the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *advance(const uint32_t *batch_end = nullptr) noexcept; - /** - * Get the remaining length of the document, from the start of the current token. - */ - simdjson_inline size_t remaining_len() const noexcept; - /** - * Check if we are at the end of the document. - * - * If this is true, there are no more tokens. - */ - simdjson_inline bool at_eof() const noexcept; - /** - * Check if we are at the beginning of the document. - */ - simdjson_inline bool at_beginning() const noexcept; - simdjson_inline uint8_t last_structural() const noexcept; - - /** - * Log that a value has been found. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_value(const char *type) const noexcept; - /** - * Log the start of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_start_value(const char *type) const noexcept; - /** - * Log the end of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_end_value(const char *type) const noexcept; - /** - * Log an error. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_error(const char *error) const noexcept; - - template - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; - template - simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; -}; - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { - logger::log_start(); - - // - // Start the document - // - if (at_eof()) { return EMPTY; } - // Only used when BOUNDED (see advance()); a local, so that it can stay in a - // register, and vanishes from the other walks. - const uint32_t *const batch_end = &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - log_start_value("document"); - SIMDJSON_TRY( visitor.visit_document_start(*this) ); - - // - // Read first value - // - { - auto value = advance(batch_end); - - // Make sure the outer object or array is closed before continuing; otherwise, there are ways we - // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 - if (!STREAMING) { - switch (*value) { - case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; - case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; - } - } - - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; - } - } - goto document_end; - -// -// Object parser states -// -object_begin: - log_start_value("object"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = false; - SIMDJSON_TRY( visitor.visit_object_start(*this) ); - - { - auto key = advance(batch_end); - if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.increment_count(*this) ); - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - -object_field: - if (simdjson_unlikely( (*advance(batch_end) != ':') )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -object_continue: - switch (*advance(batch_end)) { - case ',': - SIMDJSON_TRY( visitor.increment_count(*this) ); - { - auto key = advance(batch_end); - if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - goto object_field; - case '}': log_end_value("object"); SIMDJSON_TRY( visitor.visit_object_end(*this) ); goto scope_end; - default: log_error("No comma between object fields"); return TAPE_ERROR; - } - -scope_end: - depth--; - if (depth == 0) { goto document_end; } - if (dom_parser.is_array[depth]) { goto array_continue; } - goto object_continue; - -// -// Array parser states -// -array_begin: - log_start_value("array"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = true; - SIMDJSON_TRY( visitor.visit_array_start(*this) ); - SIMDJSON_TRY( visitor.increment_count(*this) ); - -array_value: - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -array_continue: - switch (*advance(batch_end)) { - case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; - case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; - default: log_error("Missing comma between array values"); return TAPE_ERROR; - } - -document_end: - log_end_value("document"); - SIMDJSON_TRY( visitor.visit_document_end(*this) ); - - dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); - - // If we didn't make it to the end, it's an error - if ( !STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes ) { - log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); - return TAPE_ERROR; - } - - return SUCCESS; - -} // walk_document() - -simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { -} - -// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires -// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). -// -// When BOUNDED (see dom_parser_implementation::_bounded_batch), a document may -// not run past batch_end, one past the last structural index of the batch: -// past it, we return the same dummy, and walk_document fails cleanly. -template -simdjson_inline const uint8_t *json_iterator::peek(const uint32_t *batch_end) const noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -template -simdjson_inline const uint8_t *json_iterator::advance(const uint32_t *batch_end) noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural++); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -simdjson_inline size_t json_iterator::remaining_len() const noexcept { - return dom_parser.len - *(next_structural-1); -} - -simdjson_inline bool json_iterator::at_eof() const noexcept { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; -} -simdjson_inline bool json_iterator::at_beginning() const noexcept { - return next_structural == dom_parser.structural_indexes.get(); -} -simdjson_inline uint8_t json_iterator::last_structural() const noexcept { - return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; -} - -simdjson_inline void json_iterator::log_value(const char *type) const noexcept { - logger::log_line(*this, "", type, ""); -} - -simdjson_inline void json_iterator::log_start_value(const char *type) const noexcept { - logger::log_line(*this, "+", type, ""); - if (logger::LOG_ENABLED) { logger::log_depth++; } -} - -simdjson_inline void json_iterator::log_end_value(const char *type) const noexcept { - if (logger::LOG_ENABLED) { logger::log_depth--; } - logger::log_line(*this, "-", type, ""); -} - -simdjson_inline void json_iterator::log_error(const char *error) const noexcept { - logger::log_line(*this, "", "ERROR", error); -} - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { - switch (*value) { - case '"': return visitor.visit_root_string(*this, value); - case 't': return visitor.visit_root_true_atom(*this, value); - case 'f': return visitor.visit_root_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_root_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_root_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_root_inf_atom(*this, value); -#else - case 'n': return visitor.visit_root_null_atom(*this, value); -#endif - case '-': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return visitor.visit_root_number(*this, value); - default: - log_error("Document starts with a non-value character"); - return TAPE_ERROR; - } -} -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { - // Use the fact that most scalars are going to be either strings or numbers. - if(*value == '"') { - return visitor.visit_string(*this, value); - } else if (((*value - '0') < 10) || (*value == '-')) { - return visitor.visit_number(*this, value); - } - // true, false, null are uncommon. - switch (*value) { - case 't': return visitor.visit_true_atom(*this, value); - case 'f': return visitor.visit_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_inf_atom(*this, value); -#else - case 'n': return visitor.visit_null_atom(*this, value); -#endif - default: - log_error("Non-value found when value was expected!"); - return TAPE_ERROR; - } -} - -} // namespace stage2 -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H -/* end file generic/stage2/json_iterator.h for haswell */ -/* including generic/stage2/stringparsing.h for haswell: #include */ -/* begin file generic/stage2/stringparsing.h for haswell */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace haswell { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - // Decode adjacent Unicode escapes without returning to the - // quote-and-backslash scanner between code points. - do { - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } while (src[0] == '\\' && src[1] == 'u'); - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -/** - * Bounds-safe variant of parse_string for input buffers that are NOT padded to - * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input - * byte (buf + len). It behaves exactly like parse_string while we are at least - * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays - * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few - * remaining bytes into a space-padded scratch buffer and finishes with the - * regular parse_string. This keeps the delicate escape/Unicode handling in one - * place (the proven parse_string) rather than duplicating it. - * - * Correctness relies on stage 1 having validated the string, i.e. there is an - * unescaped closing quote within [src, buf_end); that quote is therefore inside - * the copied scratch, so parse_string finds it without running off the scratch. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { - // Far from the end: identical to parse_string's loop. The guard uses - // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past - // buf_end; escape/Unicode look-aheads read within the string (before the - // closing quote, which is < buf_end), so they are in bounds here too. - // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: - // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate - // following bytes (+6 from the '\'), then (if it sees \u) another - // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the - // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the - // chunk start. The +12 margin ensures that even on kernels where - // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays - // in-bounds. The scratch fallback (3*PAD) is already safe. - while (src + SIMDJSON_PADDING + 12 <= buf_end) { - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - if (bs_quote.has_quote_first()) { - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - if (escape_char == 'u') { - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - // Within the final SIMDJSON_PADDING bytes: copy what remains into a - // space-padded scratch (spaces are neither quote nor backslash, so they do not - // disturb matching) and let the regular parser finish from there. The closing - // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in - // the chunk starting at some offset <= remaining and reads at most - // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. - // We size at 3x for a comfortable margin (the unicode look-ahead reads a few - // extra bytes past an escape). - uint8_t scratch[SIMDJSON_PADDING * 3]; - const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING - std::memset(scratch, ' ', sizeof(scratch)); - std::memcpy(scratch, src, remaining); - return parse_string(scratch, dst, allow_replacement); -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for haswell */ -/* including generic/stage2/tape_builder.h for haswell: #include */ -/* begin file generic/stage2/tape_builder.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - - -namespace simdjson { -namespace haswell { -namespace { -namespace stage2 { - -template -struct tape_builder_impl { - /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; - /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator &iter) noexcept; - - /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator &iter) noexcept; - /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator &iter) noexcept; - /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator &iter) noexcept; - - /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator &iter) noexcept; - /** - * Called when a key in a field is encountered. - * - * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array - * will be called after this with the field value. - */ - simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; - /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator &iter) noexcept; - /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator &iter) noexcept; - - /** - * Called when a string, number, boolean or null is found. - */ - simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; - /** - * Called when a string, number, boolean or null is found at the top level of a document (i.e. - * when there is no array or object and the entire document is a single string, number, boolean or - * null. - * - * This is separate from primitive() because simdjson's normal primitive parsing routines assume - * there is at least one more token after the value, which is only true in an array or object. - */ - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - -#if SIMDJSON_ENABLE_NAN_INF - simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, - // this returns a tape error on failure. - simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; -#endif - - /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; - - /** Next location to write to tape */ - tape_writer tape; -public: - simdjson_inline tape_builder_impl(dom::document &doc) noexcept; -private: - /** Next write location in the string buf for stage 2 parsing */ - uint8_t *current_string_buf_loc; - - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - simdjson_inline void start_container(json_iterator &iter) noexcept; - simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder_impl - -// Thin, non-templated entry so each architecture's stage2() keeps calling -// tape_builder::parse_document unchanged. It chooses the bounds-safe -// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the -// choice is a compile-time constant inside the walk: the padded path carries no -// extra branch or load (see tape_builder_impl::visit_string). -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } - - // The stage 2 entry points of dom::document_stream, which picks one per batch - // (see internal::dom_parser_implementation::stage2_next_function). Out of - // line, so that stage2_next() can share the code. - simdjson_warn_unused static simdjson_never_inline error_code parse_next_document( - internal::dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - return parse_document(static_cast(dom_parser), doc); - } - - // A document of a stream batch that stage 2 must bound to its structural - // indexes (see internal::dom_parser_implementation::_bounded_batch). - simdjson_warn_unused static simdjson_never_inline error_code parse_next_bounded_document( - internal::dom_parser_implementation &_dom_parser, dom::document &doc) noexcept { - dom_parser_implementation &dom_parser = static_cast(_dom_parser); - dom_parser.doc = &doc; - json_iterator iter(dom_parser, dom_parser.next_structural_index); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } -}; - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_root_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { - constexpr uint32_t start_tape_index = 0; - tape.append(start_tape_index, internal::tape_type::ROOT); - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { - return visit_string(iter, key, true); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 - return SUCCESS; -} - -template -simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { - iter.log_value(key ? "key" : "string"); - uint8_t *dst = on_start_string(iter); - // We do not allow replacement when the escape characters are invalid. - // UNPADDED is a compile-time constant chosen once per document by - // tape_builder::parse_document, so the padded build instantiates only the - // plain parse_string call below -- no runtime branch and no flag load. - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); - } else { - dst = stringparsing::parse_string(value+1, dst, false); - } - if (dst == nullptr) { - iter.log_error("Invalid escape in string"); - return STRING_ERROR; - } - on_end_string(dst); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { - return visit_string(iter, value); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("number"); - const uint8_t *num = value; - std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - // numberparsing reads ahead in 8-byte blocks for floats - // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a - // number whose digits reach the final bytes of an unpadded buffer would read - // past it. *(next_structural) is the offset of the token following this - // number, hence an upper bound on where the digits end; when that is within - // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail - // (mirroring visit_root_number). This fires only for numbers near the end. - if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { - const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document - copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, rl); - std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); - num = copy.get(); - } - } - error_code err = numberparsing::parse_number(num, tape); - if (simdjson_unlikely(err == BIGINT_ERROR && - iter.dom_parser._number_as_string)) { - // Write big integer to string buffer using the same format as strings. - // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = num; - if (*p == '-') p++; - while (numberparsing::is_digit(*p)) p++; - // The digit run must be terminated by a structural or whitespace character; otherwise the - // token is malformed (e.g. "123456789123456789123x"). - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - num); - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); - uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, num, len); - dst += len; - on_end_string(dst); - return SUCCESS; - } - return err; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { - // - // We need to make a copy to make sure that the string is space terminated. - // This is not about padding the input, which should already padded up - // to len + SIMDJSON_PADDING. However, we have no control at this stage - // on how the padding was done. What if the input string was padded with nulls? - // It is quite common for an input string to have an extra null character (C string). - // We do not want to allow 9\0 (where \0 is the null character) inside a JSON - // document, but the string "9\0" by itself is fine. So we make a copy and - // pad the input with spaces when we know that there is just one input element. - // This copy is relatively expensive, but it will almost never be called in - // practice unless you are in the strange scenario where you have many JSON - // documents made of single atoms. - // - // In a stream, the input goes on with other documents: copy up to the next - // structural only, not to the end of the batch. - const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); - std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, len); - std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); - error_code error = visit_number(iter, copy.get()); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated - // token at the very end of an unpadded buffer would over-read. Use the - // length-aware form there (the root variant already does this). - const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) - : atomparsing::is_valid_true_atom(value); - if (!ok) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) - : atomparsing::is_valid_false_atom(value); - if (!ok) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) - : atomparsing::is_valid_null_atom(value); - if (!ok) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -#if SIMDJSON_ENABLE_NAN_INF -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - // For unpadded input use the length-aware validator so the 'infinity'-style - // 8-byte compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) - : atomparsing::is_valid_nan_atom(value); - if (!ok) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. - // For unpadded input use the length-aware validator so the 'infinity' 8-byte - // compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) - : atomparsing::is_valid_inf_atom(value); - if (!ok) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure - if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -// private: - -template -simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { - return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - auto start_index = next_tape_index(iter); - tape.append(start_index+2, start); - tape.append(start_index, end); - return SUCCESS; -} - -template -simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); - iter.dom_parser.open_containers[iter.depth].count = 0; - tape.skip(); // We don't actually *write* the start element until the end. -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - // Write the ending tape element, pointing at the start location - const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; - tape.append(start_tape_index, end); - // Write the start tape element, pointing at the end location (and including count) - // count can overflow if it exceeds 24 bits... so we saturate - // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). - const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; - const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); - return SUCCESS; -} - -template -simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { - // we advance the point, accounting for the fact that we have a NULL termination - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); - return current_string_buf_loc + sizeof(uint32_t); -} - -template -simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { - uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); - // TODO check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); - // NULL termination is still handy if you expect all your strings to - // be NULL terminated? It comes at a small cost - *dst = 0; - current_string_buf_loc = dst + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H -/* end file generic/stage2/tape_builder.h for haswell */ -/* end file generic/stage2/amalgamated.h for haswell */ - -// -// Stage 1 -// - -namespace simdjson { -namespace haswell { - -simdjson_warn_unused error_code implementation::create_dom_parser_implementation( - size_t capacity, - size_t max_depth, - std::unique_ptr& dst -) const noexcept { - dst.reset( new (std::nothrow) dom_parser_implementation() ); - if (!dst) { return MEMALLOC; } - if (auto err = dst->set_capacity(capacity)) - return err; - if (auto err = dst->set_max_depth(max_depth)) - return err; - return SUCCESS; -} - -namespace { - -using namespace simd; - -// This identifies structural characters (comma, colon, braces, brackets), -// and ASCII white-space ('\r','\n','\t',' '). -simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why - // we can't use the generic lookup_16. - const auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); - - // The 6 operators (:,[]{}) have these values: - // - // , 2C - // : 3A - // [ 5B - // { 7B - // ] 5D - // } 7D - // - // If you use | 0x20 to turn [ and ] into { and }, the lower 4 bits of each character is unique. - // We exploit this, using a simd 4-bit lookup to tell us which character match against, and then - // match it (against | 0x20). - // - // To prevent recognizing other characters, everything else gets compared with 0, which cannot - // match due to the | 0x20. - // - // NOTE: Due to the | 0x20, this ALSO treats and (control characters 0C and 1A) like , - // and :. This gets caught in stage 2, which checks the actual character to ensure the right - // operators are in the right places. - const auto op_table = simd8::repeat_16( - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, ':', '{', // : = 3A, [ = 5B, { = 7B - ',', '}', 0, 0 // , = 2C, ] = 5D, } = 7D - ); - - // We compute whitespace and op separately. If later code only uses one or the - // other, given the fact that all functions are aggressively inlined, we can - // hope that useless computations will be omitted. This is namely case when - // minifying (we only need whitespace). - - const uint64_t whitespace = in.eq({ - _mm256_shuffle_epi8(whitespace_table, in.chunks[0]), - _mm256_shuffle_epi8(whitespace_table, in.chunks[1]) - }); - // Turn [ and ] into { and } - const simd8x64 curlified{ - in.chunks[0] | 0x20, - in.chunks[1] | 0x20 - }; - const uint64_t op = curlified.eq({ - _mm256_shuffle_epi8(op_table, in.chunks[0]), - _mm256_shuffle_epi8(op_table, in.chunks[1]) - }); - - return { whitespace, op }; -} - -simdjson_inline bool is_ascii(const simd8x64& input) { - return input.reduce_or().is_ascii(); -} - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be >= 0x80 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be >= 0x80 - return is_third_byte | is_fourth_byte; -} - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -// -// Stage 2 -// - -// -// Implementation-specific overrides -// -namespace simdjson { -namespace haswell { - -simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { - return haswell::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, stage1_mode streaming) noexcept { - this->buf = _buf; - this->len = _len; - return haswell::stage1::json_structural_indexer::index<128>(_buf, _len, *this, streaming); -} - -simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return haswell::stage1::generic_validate_utf8(buf,len); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_next_document(*this, _doc); -} - -internal::dom_parser_implementation::stage2_next_function_t dom_parser_implementation::stage2_next_function(bool bounded) const noexcept { - return bounded ? stage2::tape_builder::parse_next_bounded_document : stage2::tape_builder::parse_next_document; -} - -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { - return haswell::stringparsing::parse_string(src, dst, replacement_char); -} - -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { - return haswell::stringparsing::parse_wobbly_string(src, dst); -} - -simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { - auto error = stage1(_buf, _len, stage1_mode::regular); - if (error) { return error; } - return stage2(_doc); -} - -} // namespace haswell -} // namespace simdjson - -/* including simdjson/haswell/end.h: #include */ -/* begin file simdjson/haswell/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_HASWELL -SIMDJSON_UNTARGET_REGION -#endif - -/* undefining SIMDJSON_IMPLEMENTATION from "haswell" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/haswell/end.h */ - -#endif // SIMDJSON_SRC_HASWELL_CPP -/* end file haswell.cpp */ -#endif -#if SIMDJSON_IMPLEMENTATION_ICELAKE -/* including icelake.cpp: #include */ -/* begin file icelake.cpp */ -#ifndef SIMDJSON_SRC_ICELAKE_CPP -#define SIMDJSON_SRC_ICELAKE_CPP - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* including simdjson/icelake.h: #include */ -/* begin file simdjson/icelake.h */ -#ifndef SIMDJSON_ICELAKE_H -#define SIMDJSON_ICELAKE_H - -/* including simdjson/icelake/begin.h: #include "simdjson/icelake/begin.h" */ -/* begin file simdjson/icelake/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "icelake" */ -#define SIMDJSON_IMPLEMENTATION icelake -/* including simdjson/icelake/base.h: #include "simdjson/icelake/base.h" */ -/* begin file simdjson/icelake/base.h */ -#ifndef SIMDJSON_ICELAKE_BASE_H -#define SIMDJSON_ICELAKE_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_ICELAKE -namespace simdjson { -/** - * Implementation for Icelake (Intel AVX512). - */ -namespace icelake { - -class implementation; - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_BASE_H -/* end file simdjson/icelake/base.h */ -/* including simdjson/icelake/intrinsics.h: #include "simdjson/icelake/intrinsics.h" */ -/* begin file simdjson/icelake/intrinsics.h */ -#ifndef SIMDJSON_ICELAKE_INTRINSICS_H -#define SIMDJSON_ICELAKE_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - * e.g., if __AVX2__ is set... in turn, we normally set these - * macros by compiling against the corresponding architecture - * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole - * software with these advanced instructions. In simdjson, we - * want to compile the whole program for a generic target, - * and only target our specific kernels. As a workaround, - * we directly include the needed headers. These headers would - * normally guard against such usage, but we carefully included - * (or ) before, so the headers - * are fooled. - */ -#include // for _blsr_u64 -#include // for __lzcnt64 -#include // for most things (AVX2, AVX512, _popcnt64) -#include -#include -#include -#include -#include // for _mm_clmulepi64_si128 -// Important: we need the AVX-512 headers: -#include -#include -#include -#include -#include -#include -#include -// unfortunately, we may not get _blsr_u64, but, thankfully, clang -// has it as a macro. -#ifndef _blsr_u64 -// we roll our own -#define _blsr_u64(n) ((n - 1) & n) -#endif // _blsr_u64 -#endif // SIMDJSON_CLANG_VISUAL_STUDIO - -static_assert(sizeof(__m512i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for icelake"); - -#endif // SIMDJSON_ICELAKE_INTRINSICS_H -/* end file simdjson/icelake/intrinsics.h */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_ICELAKE -SIMDJSON_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi,avx512vbmi2,avx512vl,avx2,bmi,pclmul,lzcnt,popcnt") -#endif - -/* including simdjson/icelake/bitmanipulation.h: #include "simdjson/icelake/bitmanipulation.h" */ -/* begin file simdjson/icelake/bitmanipulation.h */ -#ifndef SIMDJSON_ICELAKE_BITMANIPULATION_H -#define SIMDJSON_ICELAKE_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return (int)_tzcnt_u64(input_num); -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - //////// - // You might expect the next line to be equivalent to - // return (int)_tzcnt_u64(input_num); - // but the generated code differs and might be less efficient? - //////// - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return _blsr_u64(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { - return int(_lzcnt_u64(input_num)); -} - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows - return __popcnt64(input_num);// Visual Studio wants two underscores -} -#else -simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); -} -#endif - - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_BITMANIPULATION_H -/* end file simdjson/icelake/bitmanipulation.h */ -/* including simdjson/icelake/bitmask.h: #include "simdjson/icelake/bitmask.h" */ -/* begin file simdjson/icelake/bitmask.h */ -#ifndef SIMDJSON_ICELAKE_BITMASK_H -#define SIMDJSON_ICELAKE_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { - // There should be no such thing with a processor supporting avx2 - // but not clmul. - __m128i all_ones = _mm_set1_epi8('\xFF'); - __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); - return _mm_cvtsi128_si64(result); -} - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_BITMASK_H -/* end file simdjson/icelake/bitmask.h */ -/* including simdjson/icelake/simd.h: #include "simdjson/icelake/simd.h" */ -/* begin file simdjson/icelake/simd.h */ -#ifndef SIMDJSON_ICELAKE_SIMD_H -#define SIMDJSON_ICELAKE_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if defined(__GNUC__) && !defined(__clang__) -#if __GNUC__ == 8 -#define SIMDJSON_GCC8 1 -#endif // __GNUC__ == 8 -#endif // defined(__GNUC__) && !defined(__clang__) - -#if SIMDJSON_GCC8 -/** - * GCC 8 fails to provide _mm512_set_epi8. We roll our own. - */ -inline __m512i _mm512_set_epi8(uint8_t a0, uint8_t a1, uint8_t a2, uint8_t a3, uint8_t a4, uint8_t a5, uint8_t a6, uint8_t a7, uint8_t a8, uint8_t a9, uint8_t a10, uint8_t a11, uint8_t a12, uint8_t a13, uint8_t a14, uint8_t a15, uint8_t a16, uint8_t a17, uint8_t a18, uint8_t a19, uint8_t a20, uint8_t a21, uint8_t a22, uint8_t a23, uint8_t a24, uint8_t a25, uint8_t a26, uint8_t a27, uint8_t a28, uint8_t a29, uint8_t a30, uint8_t a31, uint8_t a32, uint8_t a33, uint8_t a34, uint8_t a35, uint8_t a36, uint8_t a37, uint8_t a38, uint8_t a39, uint8_t a40, uint8_t a41, uint8_t a42, uint8_t a43, uint8_t a44, uint8_t a45, uint8_t a46, uint8_t a47, uint8_t a48, uint8_t a49, uint8_t a50, uint8_t a51, uint8_t a52, uint8_t a53, uint8_t a54, uint8_t a55, uint8_t a56, uint8_t a57, uint8_t a58, uint8_t a59, uint8_t a60, uint8_t a61, uint8_t a62, uint8_t a63) { - return _mm512_set_epi64(uint64_t(a7) + (uint64_t(a6) << 8) + (uint64_t(a5) << 16) + (uint64_t(a4) << 24) + (uint64_t(a3) << 32) + (uint64_t(a2) << 40) + (uint64_t(a1) << 48) + (uint64_t(a0) << 56), - uint64_t(a15) + (uint64_t(a14) << 8) + (uint64_t(a13) << 16) + (uint64_t(a12) << 24) + (uint64_t(a11) << 32) + (uint64_t(a10) << 40) + (uint64_t(a9) << 48) + (uint64_t(a8) << 56), - uint64_t(a23) + (uint64_t(a22) << 8) + (uint64_t(a21) << 16) + (uint64_t(a20) << 24) + (uint64_t(a19) << 32) + (uint64_t(a18) << 40) + (uint64_t(a17) << 48) + (uint64_t(a16) << 56), - uint64_t(a31) + (uint64_t(a30) << 8) + (uint64_t(a29) << 16) + (uint64_t(a28) << 24) + (uint64_t(a27) << 32) + (uint64_t(a26) << 40) + (uint64_t(a25) << 48) + (uint64_t(a24) << 56), - uint64_t(a39) + (uint64_t(a38) << 8) + (uint64_t(a37) << 16) + (uint64_t(a36) << 24) + (uint64_t(a35) << 32) + (uint64_t(a34) << 40) + (uint64_t(a33) << 48) + (uint64_t(a32) << 56), - uint64_t(a47) + (uint64_t(a46) << 8) + (uint64_t(a45) << 16) + (uint64_t(a44) << 24) + (uint64_t(a43) << 32) + (uint64_t(a42) << 40) + (uint64_t(a41) << 48) + (uint64_t(a40) << 56), - uint64_t(a55) + (uint64_t(a54) << 8) + (uint64_t(a53) << 16) + (uint64_t(a52) << 24) + (uint64_t(a51) << 32) + (uint64_t(a50) << 40) + (uint64_t(a49) << 48) + (uint64_t(a48) << 56), - uint64_t(a63) + (uint64_t(a62) << 8) + (uint64_t(a61) << 16) + (uint64_t(a60) << 24) + (uint64_t(a59) << 32) + (uint64_t(a58) << 40) + (uint64_t(a57) << 48) + (uint64_t(a56) << 56)); -} -#endif // SIMDJSON_GCC8 - - - -namespace simdjson { -namespace icelake { -namespace { -namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m512i value; - - // Zero constructor - simdjson_inline base() : value{__m512i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m512i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m512i&() const { return this->value; } - simdjson_inline operator __m512i&() { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm512_or_si512(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm512_and_si512(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm512_xor_si512(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm512_andnot_si512(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8: base> { - typedef uint32_t bitmask_t; - typedef uint64_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m512i _value) : base>(_value) {} - - friend simdjson_really_inline uint64_t operator==(const simd8 lhs, const simd8 rhs) { - return _mm512_cmpeq_epi8_mask(lhs, rhs); - } - static const int SIZE = sizeof(base::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - // workaround for compilers unable to figure out that 16 - N is a constant (GCC 8) - constexpr int shift = 16 - N; - return _mm512_alignr_epi8(*this, _mm512_permutex2var_epi64(prev_chunk, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), *this), shift); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm512_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m512i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - simdjson_inline bool any() const { return !!_mm512_test_epi8_mask (*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { return _mm512_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm512_setzero_si512(); } - static simdjson_inline simd8 load(const T values[64]) { - return _mm512_loadu_si512(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m512i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[64]) const { return _mm512_storeu_si512(reinterpret_cast<__m512i *>(dst), *this); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm512_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm512_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm512_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 64 - count_ones(mask) bytes of the result are significant but 64 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint64_t mask, L * output) const { - // we deliberately avoid _mm512_mask_compressstoreu_epi8 for portability - // (AMD Zen4 has terrible performance with it, it is effectively broken) - // _mm512_mask_compressstoreu_epi8 (output,~mask,*this); - __m512i compressed = _mm512_maskz_compress_epi8(~mask, *this); - _mm512_storeu_si512(output, compressed); // could use a mask - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[64]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, - int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, - int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31, - int8_t v32, int8_t v33, int8_t v34, int8_t v35, int8_t v36, int8_t v37, int8_t v38, int8_t v39, - int8_t v40, int8_t v41, int8_t v42, int8_t v43, int8_t v44, int8_t v45, int8_t v46, int8_t v47, - int8_t v48, int8_t v49, int8_t v50, int8_t v51, int8_t v52, int8_t v53, int8_t v54, int8_t v55, - int8_t v56, int8_t v57, int8_t v58, int8_t v59, int8_t v60, int8_t v61, int8_t v62, int8_t v63 - ) : simd8(_mm512_set_epi8( - v63, v62, v61, v60, v59, v58, v57, v56, - v55, v54, v53, v52, v51, v50, v49, v48, - v47, v46, v45, v44, v43, v42, v41, v40, - v39, v38, v37, v36, v35, v34, v33, v32, - v31, v30, v29, v28, v27, v26, v25, v24, - v23, v22, v21, v20, v19, v18, v17, v16, - v15, v14, v13, v12, v11, v10, v9, v8, - v7, v6, v5, v4, v3, v2, v1, v0 - )) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epi8(*this, other); } - - simdjson_inline simd8 operator>(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(*this, other),_mm512_set1_epi8(uint8_t(0x80))); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(other, *this),_mm512_set1_epi8(uint8_t(0x80))); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[64]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, - uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, - uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31, - uint8_t v32, uint8_t v33, uint8_t v34, uint8_t v35, uint8_t v36, uint8_t v37, uint8_t v38, uint8_t v39, - uint8_t v40, uint8_t v41, uint8_t v42, uint8_t v43, uint8_t v44, uint8_t v45, uint8_t v46, uint8_t v47, - uint8_t v48, uint8_t v49, uint8_t v50, uint8_t v51, uint8_t v52, uint8_t v53, uint8_t v54, uint8_t v55, - uint8_t v56, uint8_t v57, uint8_t v58, uint8_t v59, uint8_t v60, uint8_t v61, uint8_t v62, uint8_t v63 - ) : simd8(_mm512_set_epi8( - v63, v62, v61, v60, v59, v58, v57, v56, - v55, v54, v53, v52, v51, v50, v49, v48, - v47, v46, v45, v44, v43, v42, v41, v40, - v39, v38, v37, v36, v35, v34, v33, v32, - v31, v30, v29, v28, v27, v26, v25, v24, - v23, v22, v21, v20, v19, v18, v17, v16, - v15, v14, v13, v12, v11, v10, v9, v8, - v7, v6, v5, v4, v3, v2, v1, v0 - )) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm512_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm512_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epu8(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline uint64_t operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline uint64_t operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return _mm512_mask_blend_epi8(*this == uint8_t(0), _mm512_set1_epi8(0), _mm512_set1_epi8(-1)); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - - simdjson_inline bool is_ascii() const { return _mm512_movepi8_mask(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { - return !_mm512_test_epi8_mask(*this, *this); - } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return !_mm512_test_epi8_mask(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm512_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm512_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7-N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 1, "Icelake kernel should use one register per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} - simdjson_inline simd8x64(const simd8 chunk0) : chunks{chunk0} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr)} {} - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - this->chunks[0].compress(mask, output); - return 64 - count_ones(mask); - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - } - - simdjson_inline simd8 reduce_or() const { - return this->chunks[0]; - } - - simdjson_inline simd8x64 bit_or(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] | mask - ); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return this->chunks[0] == mask; - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return this->chunks[0] == other.chunks[0]; - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return this->chunks[0] <= mask; - } - }; // struct simd8x64 - -} // namespace simd - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_SIMD_H -/* end file simdjson/icelake/simd.h */ -/* including simdjson/icelake/stringparsing_defs.h: #include "simdjson/icelake/stringparsing_defs.h" */ -/* begin file simdjson/icelake/stringparsing_defs.h */ -#ifndef SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H -#define SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 64; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint64_t bs_bits; - uint64_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 15 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - // store to dest unconditionally - we can overwrite the bits we don't like later - v.store(dst); - return { - static_cast(v == '\\'), // bs_bits - static_cast(v == '"'), // quote_bits - }; -} - - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H -/* end file simdjson/icelake/stringparsing_defs.h */ -/* including simdjson/icelake/numberparsing_defs.h: #include "simdjson/icelake/numberparsing_defs.h" */ -/* begin file simdjson/icelake/numberparsing_defs.h */ -#ifndef SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H -#define SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace numberparsing { - -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - // this actually computes *16* values so we are being wasteful. - const __m128i ascii0 = _mm_set1_epi8('0'); - const __m128i mul_1_10 = - _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); - const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); - const __m128i mul_1_10000 = - _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); - const __m128i input = _mm_sub_epi8( - _mm_loadu_si128(reinterpret_cast(chars)), ascii0); - const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); - const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); - const __m128i t3 = _mm_packus_epi32(t2, t2); - const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); - return _mm_cvtsi128_si32( - t4); // only captures the sum of the first 8 digits, drop the rest -} - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace icelake -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H -/* end file simdjson/icelake/numberparsing_defs.h */ -/* end file simdjson/icelake/begin.h */ -/* including simdjson/generic/amalgamated.h for icelake: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for icelake */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for icelake: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for icelake */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for icelake */ -/* including simdjson/generic/jsoncharutils.h for icelake: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for icelake */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for icelake */ -/* including simdjson/generic/atomparsing.h for icelake: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for icelake */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace icelake { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for icelake */ -/* including simdjson/generic/dom_parser_implementation.h for icelake: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for icelake */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace icelake -} // namespace simdjson - -namespace simdjson { -namespace icelake { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for icelake */ -/* including simdjson/generic/implementation_simdjson_result_base.h for icelake: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for icelake */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for icelake */ -/* including simdjson/generic/numberparsing.h for icelake: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for icelake */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace icelake { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for icelake */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for icelake: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for icelake */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for icelake */ -/* end file simdjson/generic/amalgamated.h for icelake */ -/* including simdjson/icelake/end.h: #include "simdjson/icelake/end.h" */ -/* begin file simdjson/icelake/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_ICELAKE -SIMDJSON_UNTARGET_REGION -#endif - -/* undefining SIMDJSON_IMPLEMENTATION from "icelake" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/icelake/end.h */ - -#endif // SIMDJSON_ICELAKE_H -/* end file simdjson/icelake.h */ -/* including simdjson/icelake/implementation.h: #include */ -/* begin file simdjson/icelake/implementation.h */ -#ifndef SIMDJSON_ICELAKE_IMPLEMENTATION_H -#define SIMDJSON_ICELAKE_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_ICELAKE -namespace simdjson { -namespace icelake { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation( - "icelake", - "Intel/AMD AVX512", - internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2 | internal::instruction_set::AVX512F | internal::instruction_set::AVX512DQ | internal::instruction_set::AVX512CD | internal::instruction_set::AVX512BW | internal::instruction_set::AVX512VL | internal::instruction_set::AVX512VBMI2 - ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_IMPLEMENTATION_H -/* end file simdjson/icelake/implementation.h */ - -// defining SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER allows us to provide our own bit_indexer::write -#define SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -/* including simdjson/icelake/begin.h: #include */ -/* begin file simdjson/icelake/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "icelake" */ -#define SIMDJSON_IMPLEMENTATION icelake -/* including simdjson/icelake/base.h: #include "simdjson/icelake/base.h" */ -/* begin file simdjson/icelake/base.h */ -#ifndef SIMDJSON_ICELAKE_BASE_H -#define SIMDJSON_ICELAKE_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_ICELAKE -namespace simdjson { -/** - * Implementation for Icelake (Intel AVX512). - */ -namespace icelake { - -class implementation; - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_BASE_H -/* end file simdjson/icelake/base.h */ -/* including simdjson/icelake/intrinsics.h: #include "simdjson/icelake/intrinsics.h" */ -/* begin file simdjson/icelake/intrinsics.h */ -#ifndef SIMDJSON_ICELAKE_INTRINSICS_H -#define SIMDJSON_ICELAKE_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - * e.g., if __AVX2__ is set... in turn, we normally set these - * macros by compiling against the corresponding architecture - * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole - * software with these advanced instructions. In simdjson, we - * want to compile the whole program for a generic target, - * and only target our specific kernels. As a workaround, - * we directly include the needed headers. These headers would - * normally guard against such usage, but we carefully included - * (or ) before, so the headers - * are fooled. - */ -#include // for _blsr_u64 -#include // for __lzcnt64 -#include // for most things (AVX2, AVX512, _popcnt64) -#include -#include -#include -#include -#include // for _mm_clmulepi64_si128 -// Important: we need the AVX-512 headers: -#include -#include -#include -#include -#include -#include -#include -// unfortunately, we may not get _blsr_u64, but, thankfully, clang -// has it as a macro. -#ifndef _blsr_u64 -// we roll our own -#define _blsr_u64(n) ((n - 1) & n) -#endif // _blsr_u64 -#endif // SIMDJSON_CLANG_VISUAL_STUDIO - -static_assert(sizeof(__m512i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for icelake"); - -#endif // SIMDJSON_ICELAKE_INTRINSICS_H -/* end file simdjson/icelake/intrinsics.h */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_ICELAKE -SIMDJSON_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi,avx512vbmi2,avx512vl,avx2,bmi,pclmul,lzcnt,popcnt") -#endif - -/* including simdjson/icelake/bitmanipulation.h: #include "simdjson/icelake/bitmanipulation.h" */ -/* begin file simdjson/icelake/bitmanipulation.h */ -#ifndef SIMDJSON_ICELAKE_BITMANIPULATION_H -#define SIMDJSON_ICELAKE_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return (int)_tzcnt_u64(input_num); -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - //////// - // You might expect the next line to be equivalent to - // return (int)_tzcnt_u64(input_num); - // but the generated code differs and might be less efficient? - //////// - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return _blsr_u64(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { - return int(_lzcnt_u64(input_num)); -} - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows - return __popcnt64(input_num);// Visual Studio wants two underscores -} -#else -simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); -} -#endif - - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_BITMANIPULATION_H -/* end file simdjson/icelake/bitmanipulation.h */ -/* including simdjson/icelake/bitmask.h: #include "simdjson/icelake/bitmask.h" */ -/* begin file simdjson/icelake/bitmask.h */ -#ifndef SIMDJSON_ICELAKE_BITMASK_H -#define SIMDJSON_ICELAKE_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { - // There should be no such thing with a processor supporting avx2 - // but not clmul. - __m128i all_ones = _mm_set1_epi8('\xFF'); - __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); - return _mm_cvtsi128_si64(result); -} - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_BITMASK_H -/* end file simdjson/icelake/bitmask.h */ -/* including simdjson/icelake/simd.h: #include "simdjson/icelake/simd.h" */ -/* begin file simdjson/icelake/simd.h */ -#ifndef SIMDJSON_ICELAKE_SIMD_H -#define SIMDJSON_ICELAKE_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if defined(__GNUC__) && !defined(__clang__) -#if __GNUC__ == 8 -#define SIMDJSON_GCC8 1 -#endif // __GNUC__ == 8 -#endif // defined(__GNUC__) && !defined(__clang__) - -#if SIMDJSON_GCC8 -/** - * GCC 8 fails to provide _mm512_set_epi8. We roll our own. - */ -inline __m512i _mm512_set_epi8(uint8_t a0, uint8_t a1, uint8_t a2, uint8_t a3, uint8_t a4, uint8_t a5, uint8_t a6, uint8_t a7, uint8_t a8, uint8_t a9, uint8_t a10, uint8_t a11, uint8_t a12, uint8_t a13, uint8_t a14, uint8_t a15, uint8_t a16, uint8_t a17, uint8_t a18, uint8_t a19, uint8_t a20, uint8_t a21, uint8_t a22, uint8_t a23, uint8_t a24, uint8_t a25, uint8_t a26, uint8_t a27, uint8_t a28, uint8_t a29, uint8_t a30, uint8_t a31, uint8_t a32, uint8_t a33, uint8_t a34, uint8_t a35, uint8_t a36, uint8_t a37, uint8_t a38, uint8_t a39, uint8_t a40, uint8_t a41, uint8_t a42, uint8_t a43, uint8_t a44, uint8_t a45, uint8_t a46, uint8_t a47, uint8_t a48, uint8_t a49, uint8_t a50, uint8_t a51, uint8_t a52, uint8_t a53, uint8_t a54, uint8_t a55, uint8_t a56, uint8_t a57, uint8_t a58, uint8_t a59, uint8_t a60, uint8_t a61, uint8_t a62, uint8_t a63) { - return _mm512_set_epi64(uint64_t(a7) + (uint64_t(a6) << 8) + (uint64_t(a5) << 16) + (uint64_t(a4) << 24) + (uint64_t(a3) << 32) + (uint64_t(a2) << 40) + (uint64_t(a1) << 48) + (uint64_t(a0) << 56), - uint64_t(a15) + (uint64_t(a14) << 8) + (uint64_t(a13) << 16) + (uint64_t(a12) << 24) + (uint64_t(a11) << 32) + (uint64_t(a10) << 40) + (uint64_t(a9) << 48) + (uint64_t(a8) << 56), - uint64_t(a23) + (uint64_t(a22) << 8) + (uint64_t(a21) << 16) + (uint64_t(a20) << 24) + (uint64_t(a19) << 32) + (uint64_t(a18) << 40) + (uint64_t(a17) << 48) + (uint64_t(a16) << 56), - uint64_t(a31) + (uint64_t(a30) << 8) + (uint64_t(a29) << 16) + (uint64_t(a28) << 24) + (uint64_t(a27) << 32) + (uint64_t(a26) << 40) + (uint64_t(a25) << 48) + (uint64_t(a24) << 56), - uint64_t(a39) + (uint64_t(a38) << 8) + (uint64_t(a37) << 16) + (uint64_t(a36) << 24) + (uint64_t(a35) << 32) + (uint64_t(a34) << 40) + (uint64_t(a33) << 48) + (uint64_t(a32) << 56), - uint64_t(a47) + (uint64_t(a46) << 8) + (uint64_t(a45) << 16) + (uint64_t(a44) << 24) + (uint64_t(a43) << 32) + (uint64_t(a42) << 40) + (uint64_t(a41) << 48) + (uint64_t(a40) << 56), - uint64_t(a55) + (uint64_t(a54) << 8) + (uint64_t(a53) << 16) + (uint64_t(a52) << 24) + (uint64_t(a51) << 32) + (uint64_t(a50) << 40) + (uint64_t(a49) << 48) + (uint64_t(a48) << 56), - uint64_t(a63) + (uint64_t(a62) << 8) + (uint64_t(a61) << 16) + (uint64_t(a60) << 24) + (uint64_t(a59) << 32) + (uint64_t(a58) << 40) + (uint64_t(a57) << 48) + (uint64_t(a56) << 56)); -} -#endif // SIMDJSON_GCC8 - - - -namespace simdjson { -namespace icelake { -namespace { -namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m512i value; - - // Zero constructor - simdjson_inline base() : value{__m512i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m512i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m512i&() const { return this->value; } - simdjson_inline operator __m512i&() { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm512_or_si512(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm512_and_si512(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm512_xor_si512(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm512_andnot_si512(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8: base> { - typedef uint32_t bitmask_t; - typedef uint64_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m512i _value) : base>(_value) {} - - friend simdjson_really_inline uint64_t operator==(const simd8 lhs, const simd8 rhs) { - return _mm512_cmpeq_epi8_mask(lhs, rhs); - } - static const int SIZE = sizeof(base::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - // workaround for compilers unable to figure out that 16 - N is a constant (GCC 8) - constexpr int shift = 16 - N; - return _mm512_alignr_epi8(*this, _mm512_permutex2var_epi64(prev_chunk, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), *this), shift); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm512_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m512i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - simdjson_inline bool any() const { return !!_mm512_test_epi8_mask (*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { return _mm512_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm512_setzero_si512(); } - static simdjson_inline simd8 load(const T values[64]) { - return _mm512_loadu_si512(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m512i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[64]) const { return _mm512_storeu_si512(reinterpret_cast<__m512i *>(dst), *this); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm512_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm512_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm512_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 64 - count_ones(mask) bytes of the result are significant but 64 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint64_t mask, L * output) const { - // we deliberately avoid _mm512_mask_compressstoreu_epi8 for portability - // (AMD Zen4 has terrible performance with it, it is effectively broken) - // _mm512_mask_compressstoreu_epi8 (output,~mask,*this); - __m512i compressed = _mm512_maskz_compress_epi8(~mask, *this); - _mm512_storeu_si512(output, compressed); // could use a mask - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[64]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, - int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, - int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31, - int8_t v32, int8_t v33, int8_t v34, int8_t v35, int8_t v36, int8_t v37, int8_t v38, int8_t v39, - int8_t v40, int8_t v41, int8_t v42, int8_t v43, int8_t v44, int8_t v45, int8_t v46, int8_t v47, - int8_t v48, int8_t v49, int8_t v50, int8_t v51, int8_t v52, int8_t v53, int8_t v54, int8_t v55, - int8_t v56, int8_t v57, int8_t v58, int8_t v59, int8_t v60, int8_t v61, int8_t v62, int8_t v63 - ) : simd8(_mm512_set_epi8( - v63, v62, v61, v60, v59, v58, v57, v56, - v55, v54, v53, v52, v51, v50, v49, v48, - v47, v46, v45, v44, v43, v42, v41, v40, - v39, v38, v37, v36, v35, v34, v33, v32, - v31, v30, v29, v28, v27, v26, v25, v24, - v23, v22, v21, v20, v19, v18, v17, v16, - v15, v14, v13, v12, v11, v10, v9, v8, - v7, v6, v5, v4, v3, v2, v1, v0 - )) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epi8(*this, other); } - - simdjson_inline simd8 operator>(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(*this, other),_mm512_set1_epi8(uint8_t(0x80))); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(other, *this),_mm512_set1_epi8(uint8_t(0x80))); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[64]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, - uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, - uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31, - uint8_t v32, uint8_t v33, uint8_t v34, uint8_t v35, uint8_t v36, uint8_t v37, uint8_t v38, uint8_t v39, - uint8_t v40, uint8_t v41, uint8_t v42, uint8_t v43, uint8_t v44, uint8_t v45, uint8_t v46, uint8_t v47, - uint8_t v48, uint8_t v49, uint8_t v50, uint8_t v51, uint8_t v52, uint8_t v53, uint8_t v54, uint8_t v55, - uint8_t v56, uint8_t v57, uint8_t v58, uint8_t v59, uint8_t v60, uint8_t v61, uint8_t v62, uint8_t v63 - ) : simd8(_mm512_set_epi8( - v63, v62, v61, v60, v59, v58, v57, v56, - v55, v54, v53, v52, v51, v50, v49, v48, - v47, v46, v45, v44, v43, v42, v41, v40, - v39, v38, v37, v36, v35, v34, v33, v32, - v31, v30, v29, v28, v27, v26, v25, v24, - v23, v22, v21, v20, v19, v18, v17, v16, - v15, v14, v13, v12, v11, v10, v9, v8, - v7, v6, v5, v4, v3, v2, v1, v0 - )) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm512_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm512_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epu8(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline uint64_t operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline uint64_t operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return _mm512_mask_blend_epi8(*this == uint8_t(0), _mm512_set1_epi8(0), _mm512_set1_epi8(-1)); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - - simdjson_inline bool is_ascii() const { return _mm512_movepi8_mask(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { - return !_mm512_test_epi8_mask(*this, *this); - } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return !_mm512_test_epi8_mask(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm512_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm512_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7-N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 1, "Icelake kernel should use one register per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} - simdjson_inline simd8x64(const simd8 chunk0) : chunks{chunk0} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr)} {} - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - this->chunks[0].compress(mask, output); - return 64 - count_ones(mask); - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - } - - simdjson_inline simd8 reduce_or() const { - return this->chunks[0]; - } - - simdjson_inline simd8x64 bit_or(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] | mask - ); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return this->chunks[0] == mask; - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return this->chunks[0] == other.chunks[0]; - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return this->chunks[0] <= mask; - } - }; // struct simd8x64 - -} // namespace simd - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_SIMD_H -/* end file simdjson/icelake/simd.h */ -/* including simdjson/icelake/stringparsing_defs.h: #include "simdjson/icelake/stringparsing_defs.h" */ -/* begin file simdjson/icelake/stringparsing_defs.h */ -#ifndef SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H -#define SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 64; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint64_t bs_bits; - uint64_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 15 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - // store to dest unconditionally - we can overwrite the bits we don't like later - v.store(dst); - return { - static_cast(v == '\\'), // bs_bits - static_cast(v == '"'), // quote_bits - }; -} - - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H -/* end file simdjson/icelake/stringparsing_defs.h */ -/* including simdjson/icelake/numberparsing_defs.h: #include "simdjson/icelake/numberparsing_defs.h" */ -/* begin file simdjson/icelake/numberparsing_defs.h */ -#ifndef SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H -#define SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace numberparsing { - -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - // this actually computes *16* values so we are being wasteful. - const __m128i ascii0 = _mm_set1_epi8('0'); - const __m128i mul_1_10 = - _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); - const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); - const __m128i mul_1_10000 = - _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); - const __m128i input = _mm_sub_epi8( - _mm_loadu_si128(reinterpret_cast(chars)), ascii0); - const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); - const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); - const __m128i t3 = _mm_packus_epi32(t2, t2); - const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); - return _mm_cvtsi128_si32( - t4); // only captures the sum of the first 8 digits, drop the rest -} - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace icelake -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H -/* end file simdjson/icelake/numberparsing_defs.h */ -/* end file simdjson/icelake/begin.h */ -/* including generic/amalgamated.h for icelake: #include */ -/* begin file generic/amalgamated.h for icelake */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) -#error generic/dependencies.h must be included before generic/amalgamated.h! -#endif - -/* including generic/base.h for icelake: #include */ -/* begin file generic/base.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { - -struct json_character_block; - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_BASE_H -/* end file generic/base.h for icelake */ -/* including generic/dom_parser_implementation.h for icelake: #include */ -/* begin file generic/dom_parser_implementation.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// Interface a dom parser implementation must fulfill -namespace simdjson { -namespace icelake { -namespace { - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); -simdjson_inline bool is_ascii(const simd8x64& input); - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file generic/dom_parser_implementation.h for icelake */ -/* including generic/json_character_block.h for icelake: #include */ -/* begin file generic/json_character_block.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { - -struct json_character_block { - static simdjson_inline json_character_block classify(const simd::simd8x64& in); - - simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_inline uint64_t op() const noexcept { return _op; } - simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } - - uint64_t _whitespace; - uint64_t _op; -}; - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H -/* end file generic/json_character_block.h for icelake */ -/* end file generic/amalgamated.h for icelake */ -/* including generic/stage1/amalgamated.h for icelake: #include */ -/* begin file generic/stage1/amalgamated.h for icelake */ -// Stuff other things depend on -/* including generic/stage1/base.h for icelake: #include */ -/* begin file generic/stage1/base.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { -namespace stage1 { - -class bit_indexer; -template -struct buf_block_reader; -struct json_block; -class json_minifier; -class json_scanner; -struct json_string_block; -class json_string_scanner; -class json_structural_indexer; - -} // namespace stage1 - -namespace utf8_validation { -struct utf8_checker; -} // namespace utf8_validation - -using utf8_validation::utf8_checker; - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H -/* end file generic/stage1/base.h for icelake */ -/* including generic/stage1/buf_block_reader.h for icelake: #include */ -/* begin file generic/stage1/buf_block_reader.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace icelake { -namespace { -namespace stage1 { - -// Walks through a buffer in block-sized increments, loading the last part with spaces -template -struct buf_block_reader { -public: - simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t *full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t *dst) const; - /** - * Like get_remainder, for input followed by SIMDJSON_PADDING readable bytes. - * - * get_remainder writes the block with a memset and a memcpy of the remaining - * length, and the vector loads that read it back cannot be forwarded from - * those narrower stores: they wait until the stores reach the cache, which - * costs tens of cycles on every document. Here the block is put together in - * vector registers, from loads that may read into the padding, and written - * with full-width stores that the loads can forward from. - */ - simdjson_inline size_t get_remainder_padded(uint8_t *dst) const; - simdjson_inline void advance(); -private: - const uint8_t *buf; - const size_t len; - const size_t lenminusstep; - size_t idx; -}; - -template -simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} - -template -simdjson_inline size_t buf_block_reader::block_index() { return idx; } - -template -simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; -} - -template -simdjson_inline const uint8_t *buf_block_reader::full_block() const { - return &buf[idx]; -} - -template -simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { - if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; -} - -// keep_mask[128 - n + i] is 0xFF for i < n and 0 otherwise. -alignas(64) static const uint8_t keep_mask[256] = { - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF -}; - -template -simdjson_inline size_t buf_block_reader::get_remainder_padded(uint8_t *dst) const { - static_assert(STEP_SIZE <= 128, "keep_mask covers blocks of up to 128 bytes"); - static_assert(sizeof(simd::simd8) <= SIMDJSON_PADDING, "a vector load at the last byte must stay in the padding"); - const size_t remaining = len - idx; - if (remaining == 0) { return 0; } - const simd::simd8 spaces = simd::simd8::splat(0x20); - for (size_t off = 0; off < STEP_SIZE; off += sizeof(simd::simd8)) { - simd::simd8 v = spaces; - if (off < remaining) { - // Starts before len, so it ends within the padding. - const simd::simd8 in = simd::simd8::load(buf + idx + off); - const simd::simd8 keep = simd::simd8::load(keep_mask + 128 - remaining + off); - v = spaces ^ ((in ^ spaces) & keep); - } - v.store(dst + off); - } - return remaining; -} - -template -simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for icelake */ -/* including generic/stage1/json_escape_scanner.h for icelake: #include */ -/* begin file generic/stage1/json_escape_scanner.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { -namespace stage1 { - -/** - * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). - */ -struct json_escape_scanner { - /** The actual escape characters (the backslashes themselves). */ - uint64_t next_is_escaped = 0ULL; - - struct escaped_and_escape { - /** - * Mask of escaped characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 0100100010100101000 - * n \ \ n \ \ - * ``` - */ - uint64_t escaped; - /** - * Mask of escape characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 1001000101001010001 - * \ \ \ \ \ \ \ - * ``` - */ - uint64_t escape; - }; - - /** - * Get a mask of both escape and escaped characters (the characters following a backslash). - * - * @param potential_escape A mask of the character that can escape others (but could be - * escaped itself). e.g. block.eq('\\') - */ - simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { - -#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT - if (!backslash) { return {next_escaped_without_backslashes(), 0}; } -#endif - - // | | Mask (shows characters instead of 1's) | Depth | Instructions | - // |--------------------------------|----------------------------------------|-------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | - // | | ` even odd even odd odd` | | | - // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) - // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) - // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) - // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () - // (*) this is not needed until the next iteration - uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); - uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); - uint64_t escape = escape_and_terminal_code & backslash; - this->next_is_escaped = escape >> 63; - return {escaped, escape}; - } - -private: - static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; - - simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { - uint64_t escaped = this->next_is_escaped; - this->next_is_escaped = 0; - return escaped; - } - - /** - * Returns a mask of the next escape characters (masking out escaped backslashes), along with - * any non-backslash escape codes. - * - * \n \\n \\\n \\\\n returns: - * \n \ \ \n \ \ - * 11 100 1011 10100 - * - * You are expected to mask out the first bit yourself if the previous block had a trailing - * escape. - * - * & the result with potential_escape to get just the escape characters. - * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. - */ - static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { - // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: - // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be - // inverted (\\\ would be 010 instead of 101). - // - // ``` - // string: | ____\\\\_\\\\_____ | - // maybe_escaped | ODD | \ \ \ \ | - // even-aligned ^^^ ^^^^ odd-aligned - // ``` - // - // Taking that into account, our basic strategy is: - // - // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for - // odd-aligned runs. - // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the - // odd bits in odd-aligned runs. - // 3. & with backslash to clean up any stray bits. - // runs are set to 0, and then XORing with "odd": - // - // | | Mask (shows characters instead of 1's) | Instructions | - // |--------------------------------|----------------------------------------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | - // | | ` even odd even odd odd` | - // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) - // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) - // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) - // - - // Escaped characters are characters following an escape. - uint64_t maybe_escaped = potential_escape << 1; - - // To distinguish odd from even escape sequences, therefore, we turn on any *starting* - // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) - // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. - // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. - // - All other odd bytes are 1, and even bytes are 0. - uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; - uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; - - // Now we flip all odd bytes back with xor. This: - // - Makes odd runs of backslashes go from 0000 to 1010 - // - Makes even runs of backslashes go from 1111 to 1010 - // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) - // - Resets all other bytes to 0 - return even_series_codes_and_odd_bits ^ ODD_BITS; - } -}; - -} // namespace stage1 -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_escape_scanner.h for icelake */ -/* including generic/stage1/json_string_scanner.h for icelake: #include */ -/* begin file generic/stage1/json_string_scanner.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { -namespace stage1 { - -struct json_string_block { - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : - _escaped(escaped), _quote(quote), _in_string(in_string) {} - - // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } - // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } - // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } - // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } - - // escaped characters (backslashed--does not include the hex characters after \u) - uint64_t _escaped; - // real quotes (non-escaped ones) - uint64_t _quote; - // string characters (includes start quote but not end quote) - uint64_t _in_string; -}; - -// Scans blocks for string characters, storing the state necessary to do so -class json_string_scanner { -public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); - -private: - // Scans for escape characters - json_escape_scanner escape_scanner{}; - // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). - uint64_t prev_in_string = 0ULL; -}; - -// -// Return a mask of all string characters plus end quotes. -// -// prev_escaped is overflow saying whether the next character is escaped. -// prev_in_string is overflow saying whether we're still in a string. -// -// Backslash sequences outside of quotes will be detected in stage 2. -// -simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = escape_scanner.next(backslash).escaped; - const uint64_t quote = in.eq('"') & ~escaped; - - // - // prefix_xor flips on bits inside the string (and flips off the end quote). - // - // Then we xor with prev_in_string: if we were in a string already, its effect is flipped - // (characters inside strings are outside, and characters outside strings are inside). - // - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - - // - // Check if we're still in a string at the end of the box so the next block will know - // - prev_in_string = uint64_t(static_cast(in_string) >> 63); - - // Use ^ to turn the beginning quote off, and the end quote on. - - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_string_block(escaped, quote, in_string); -} - -simdjson_really_inline error_code json_string_scanner::finish() { - if (prev_in_string) { - return UNCLOSED_STRING; - } - return SUCCESS; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_string_scanner.h for icelake */ -/* including generic/stage1/utf8_lookup4_algorithm.h for icelake: #include */ -/* begin file generic/stage1/utf8_lookup4_algorithm.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { -namespace utf8_validation { - -using namespace simd; - - simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { -// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) -// Bit 1 = Too Long (ASCII followed by continuation) -// Bit 2 = Overlong 3-byte -// Bit 4 = Surrogate -// Bit 5 = Overlong 2-byte -// Bit 7 = Two Continuations - constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ - // 11______ 11______ - constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ - constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ - constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ - constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ - constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ - constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ - // 11110100 101_____ - // 11110101 1001____ - // 11110101 101_____ - // 1111011_ 1001____ - // 1111011_ 101_____ - // 11111___ 1001____ - // 11111___ 101_____ - constexpr const uint8_t TOO_LARGE_1000 = 1<<6; - // 11110101 1000____ - // 1111011_ 1000____ - // 11111___ 1000____ - constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ - - const simd8 byte_1_high = prev1.shr<4>().lookup_16( - // 0_______ ________ - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - // 10______ ________ - TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, - // 1100____ ________ - TOO_SHORT | OVERLONG_2, - // 1101____ ________ - TOO_SHORT, - // 1110____ ________ - TOO_SHORT | OVERLONG_3 | SURROGATE, - // 1111____ ________ - TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 - ); - constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( - // ____0000 ________ - CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, - // ____0001 ________ - CARRY | OVERLONG_2, - // ____001_ ________ - CARRY, - CARRY, - - // ____0100 ________ - CARRY | TOO_LARGE, - // ____0101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____011_ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - - // ____1___ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____1101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000 - ); - const simd8 byte_2_high = input.shr<4>().lookup_16( - // ________ 0_______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - - // ________ 1000____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, - // ________ 1001____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, - // ________ 101_____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - - // ________ 11______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT - ); - return (byte_1_high & byte_1_low & byte_2_high); - } - simdjson_inline simd8 check_multibyte_lengths(const simd8 input, - const simd8 prev_input, const simd8 sc) { - simd8 prev2 = input.prev<2>(prev_input); - simd8 prev3 = input.prev<3>(prev_input); - simd8 must23 = must_be_2_3_continuation(prev2, prev3); - simd8 must23_80 = must23 & uint8_t(0x80); - return must23_80 ^ sc; - } - - // - // Return nonzero if there are incomplete multibyte characters at the end of the block: - // e.g. if there is a 4-byte character, but it's 3 bytes from the end. - // - simdjson_inline simd8 is_incomplete(const simd8 input) { - // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): - // ... 1111____ 111_____ 11______ -#if SIMDJSON_IMPLEMENTATION_ICELAKE || (SIMDJSON_IMPLEMENTATION_RVV_VLS && __riscv_v_fixed_vlen >= 512) - static const uint8_t max_array[64] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#else - static const uint8_t max_array[32] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#endif - const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); - return input.gt_bits(max_value); - } - - struct utf8_checker { - // If this is nonzero, there has been a UTF-8 error. - simd8 error; - // The last input we received - simd8 prev_input_block; - // Whether the last input we received was incomplete (used for ASCII fast path) - simd8 prev_incomplete; - - // - // Check whether the current bytes are valid UTF-8. - // - simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { - // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes - // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) - simd8 prev1 = input.prev<1>(prev_input); - simd8 sc = check_special_cases(input, prev1); - this->error |= check_multibyte_lengths(input, prev_input, sc); - } - - // The only problem that can happen at EOF is that a multibyte character is too short - // or a byte value too large in the last bytes: check_special_cases only checks for bytes - // too large in the first of two bytes. - simdjson_inline void check_eof() { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; - } - - simdjson_inline void check_next_input(const simd8x64& input) { - if(simdjson_likely(is_ascii(input))) { - this->error |= this->prev_incomplete; - } else { - // you might think that a for-loop would work, but under Visual Studio, it is not good enough. - static_assert((simd8x64::NUM_CHUNKS == 1) - ||(simd8x64::NUM_CHUNKS == 2) - || (simd8x64::NUM_CHUNKS == 4), - "We support one, two or four chunks per 64-byte block."); - SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 1) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 2) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 4) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - this->check_utf8_bytes(input.get<2>(), input.get<1>()); - this->check_utf8_bytes(input.get<3>(), input.get<2>()); - } - this->prev_incomplete = is_incomplete(input.get::NUM_CHUNKS-1>()); - this->prev_input_block = input.get::NUM_CHUNKS-1>(); - } - } - // do not forget to call check_eof! - simdjson_warn_unused simdjson_inline error_code errors() { - return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; - } - - }; // struct utf8_checker -} // namespace utf8_validation - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H -/* end file generic/stage1/utf8_lookup4_algorithm.h for icelake */ -/* including generic/stage1/json_scanner.h for icelake: #include */ -/* begin file generic/stage1/json_scanner.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { -namespace stage1 { - -/** - * A block of scanned json, with information on operators and scalars. - * - * We seek to identify pseudo-structural characters. Anything that is inside - * a string must be omitted (hence & ~_string.string_tail()). - * Otherwise, pseudo-structural characters come in two forms. - * 1. We have the structural characters ([,],{,},:, comma). The - * term 'structural character' is from the JSON RFC. - * 2. We have the 'scalar pseudo-structural characters'. - * Scalars are quotes, and any character except structural characters and white space. - * - * To identify the scalar pseudo-structural characters, we must look at what comes - * before them: it must be a space, a quote or a structural characters. - * Starting with simdjson v0.3, we identify them by - * negation: we identify everything that is followed by a non-quote scalar, - * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. - */ -struct json_block { -public: - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - - /** - * The start of structurals. - * In simdjson prior to v0.3, these were called the pseudo-structural characters. - **/ - simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } - /** All JSON whitespace (i.e. not in a string) */ - simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } - - // Helpers - - /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } - /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } - - // string and escape characters - json_string_block _string; - // whitespace, structural characters ('operators'), scalars - json_character_block _characters; - // whether the previous character was a scalar - uint64_t _follows_potential_nonquote_scalar; -private: - // Potential structurals (i.e. disregarding strings) - - /** - * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". - * They may reside inside a string. - **/ - simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } - /** - * The start of non-operator runs, like 123, true and "abc". - * It main reside inside a string. - **/ - simdjson_inline uint64_t potential_scalar_start() const noexcept { - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space - // then we know that it is irrelevant structurally. - return _characters.scalar() & ~follows_potential_scalar(); - } - /** - * Whether the given character is immediately after a non-operator like 123, true. - * The characters following a quote are not included. - */ - simdjson_inline uint64_t follows_potential_scalar() const noexcept { - // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character - // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a - // white space. - // It is understood that within quoted region, anything at all could be marked (irrelevant). - return _follows_potential_nonquote_scalar; - } -}; - -/** - * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. - * - * The scanner starts by calculating two distinct things: - * - string characters (taking \" into account) - * - structural characters or 'operators' ([]{},:, comma) - * and scalars (runs of non-operators like 123, true and "abc") - * - * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: - * in particular, the operator/scalar bit will find plenty of things that are actually part of - * strings. When we're done, json_block will fuse the two together by masking out tokens that are - * part of a string. - */ -class json_scanner { -public: - json_scanner() = default; - simdjson_inline json_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_warn_unused simdjson_inline error_code finish(); - -private: - // Whether the last character of the previous iteration is part of a scalar token - // (anything except whitespace or a structural character/'operator'). - uint64_t prev_scalar = 0ULL; - json_string_scanner string_scanner{}; -}; - - -// -// Check if the current character immediately follows a matching character. -// -// For example, this checks for quotes with backslashes in front of them: -// -// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); -// -simdjson_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; -} - -simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { - json_string_block strings = string_scanner.next(in); - // identifies the white-space and the structural characters - json_character_block characters = json_character_block::classify(in); - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). - // - // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) - // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential - // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we - // may need to add an extra check when parsing strings. - // - // Performance: there are many ways to skin this cat. - const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); - uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_block( - strings,// strings is a function-local object so either it moves or the copy is elided. - characters, - follows_nonquote_scalar - ); -} - -simdjson_warn_unused simdjson_inline error_code json_scanner::finish() { - return string_scanner.finish(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H -/* end file generic/stage1/json_scanner.h for icelake */ - -// All other declarations -/* including generic/stage1/find_next_document_index.h for icelake: #include */ -/* begin file generic/stage1/find_next_document_index.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { -namespace stage1 { - -/** - * This algorithm is used to quickly identify the last structural position that - * makes up a complete document. - * - * It does this by going backwards and finding the last *document boundary* (a - * place where one value follows another without a comma between them). If the - * last document (the characters after the boundary) has an equal number of - * start and end brackets, it is considered complete. - * - * Simply put, we iterate over the structural characters, starting from - * the end. We consider that we found the end of a JSON document when the - * first element of the pair is NOT one of these characters: '{' '[' ':' ',' - * and when the second element is NOT one of these characters: '}' ']' ':' ','. - * - * This simple comparison works most of the time, but it does not cover cases - * where the batch's structural indexes contain a perfect amount of documents. - * In such a case, we do not have access to the structural index which follows - * the last document, therefore, we do not have access to the second element in - * the pair, and that means we cannot identify the last document. To fix this - * issue, we keep a count of the open and closed curly/square braces we found - * while searching for the pair. When we find a pair AND the count of open and - * closed curly/square braces is the same, we know that we just passed a - * complete document, therefore the last json buffer location is the end of the - * batch. - */ -simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { - const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - const uint8_t e = parser.buf[len - 1]; - return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && - e != ' ' && e != '\t' && e != '\n' && e != '\r'; -} - -/** - * The structural index past the last one of a partial batch is the end of this - * window, not a token of it: the byte there is the first byte of the next - * window. A finished value must not be rejected because that byte is ',', ':', - * ']' or '}'. A last document that itself ends with '{', '[', ',' or ':' is - * invalid JSON, yet its brackets may balance: walking it, the On-Demand API - * could read the next batch as a value. We leave such a document to the next - * batch, so that the batch ends between two documents. - */ -simdjson_inline bool ends_with_unfinished_document(dom_parser_implementation &parser) { - const uint8_t last = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - return last == '{' || last == '[' || last == ',' || last == ':'; -} - -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { - // Variant: do not count separately, just figure out depth - if(parser.n_structural_indexes == 0) { return 0; } - auto arr_cnt = 0; - auto obj_cnt = 0; - for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { - auto idxb = parser.structural_indexes[i]; - switch (parser.buf[idxb]) { - case ':': - case ',': - continue; - case '}': - obj_cnt--; - continue; - case ']': - arr_cnt--; - continue; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - auto idxa = parser.structural_indexes[i - 1]; - switch (parser.buf[idxa]) { - case '{': - case '[': - case ':': - case ',': - continue; - } - // Last document is complete, so the next document will appear after! - if (!arr_cnt && !obj_cnt) { - return defer_last ? i : parser.n_structural_indexes; - } - // Last document is incomplete; mark the document at i + 1 as the next one - return i; - } - // If we made it to the end, we want to finish counting to see if we have a full document. - switch (parser.buf[parser.structural_indexes[0]]) { - case '}': - obj_cnt--; - break; - case ']': - arr_cnt--; - break; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - if (!arr_cnt && !obj_cnt) { - // We have a complete document. - return defer_last ? 0 : parser.n_structural_indexes; - } - return 0; -} - -/** - * Sentinel value returned to indicate a document started but didn't fit - * (CAPACITY error), as opposed to 0 which means no document content found - * (EMPTY). - */ -constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; - -/** - * For RFC 7464 JSON text sequences, filter RS from structural indexes and - * find batch boundaries. - * - * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. - * RS bytes appear in structural_indexes as they are classified as scalars. - * This function: - * 1. Scans structural_indexes to find and count RS positions - * 2. Filters RS out of structural_indexes in-place - * 3. Determines batch boundaries based on RS positions - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @param scan_len Offset past which no value start may be derived. Equals len - * unless stage 1 dropped a trailing unclosed string, whose bytes it - * never validated. - * @return The number of structural indexes to keep (after RS filtering), - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t find_next_document_index_json_sequence( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start, - size_t scan_len) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Phase 1: Scan structural_indexes to find RS positions and handle them. - // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS - // is already in structural_indexes (it's an operator). For scalars like numbers, - // the digit following RS is NOT in structural_indexes because the scanner sees - // RS as a scalar, making the digit a scalar continuation, not a start. - // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the - // actual value start position. - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_rs_pos = 0; - uint32_t rs_count = 0; - - for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { - const uint32_t pos = parser.structural_indexes[read_idx]; - if (parser.buf[pos] == 0x1E) { - // This is an RS character - find the actual JSON value start. - last_rs_pos = pos; - rs_count++; - // Skip past this RS and any whitespace *and any additional RSes* - // to locate the real value. Consecutive RSes are degenerate - // "empty records" per RFC 7464; we collapse them here. They do - // not always appear as separate entries in structural_indexes - // because the scanner groups runs of adjacent non-whitespace - // scalar bytes (including RS) into a single scalar start. - uint32_t value_start = pos + 1; - while (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { - value_start++; - } else if (c == 0x1E) { - // Collapsed empty record. Still count it so rs_count reflects - // the true number of record markers and last_rs_pos tracks - // the final one. - last_rs_pos = value_start; - rs_count++; - value_start++; - } else { - break; - } - } - // If the scanner emitted additional structurals inside the - // whitespace+RS run we just walked over (i.e., isolated RSes - // separated by whitespace), skip past them so we do not - // double-count or double-emit. - while (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] < value_start) { - read_idx++; - } - // Check if the value start is an operator (always present in - // scanner structural_indexes) or a scalar-like start (which may - // be missing from structural_indexes and must be added here). - // Note: '"' is NOT always in structural_indexes. The scanner - // classifies '"' as a scalar character and emits it as a - // structural only when it is a *scalar start* (preceded by - // whitespace or an operator). When '"' immediately follows an - // RS (which the scanner also classifies as scalar), it is - // treated as a scalar continuation and not emitted - so we - // must add it here just like any other scalar value. - if (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - const bool is_operator = - (c == '{' || c == '}' || c == '[' || c == ']' || - c == ':' || c == ','); - // If the next scanner structural is exactly at value_start, - // the scanner already emitted it (it followed whitespace) and - // we must not add a duplicate - a subsequent iteration will - // copy it into write_idx. - const bool already_emitted = - (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] == value_start); - if (!is_operator && !already_emitted) { - // Scalar value (number/true/false/null/string) - add its - // position since scanner missed it. - parser.structural_indexes[write_idx++] = value_start; - } - } - } else { - // Not RS, copy to output - parser.structural_indexes[write_idx++] = pos; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { - // Only RS markers here: the last one opens a record continuing past the - // window, so that is where the next batch resumes. - if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } - return 0; - } - if (rs_count == 0) { - // No RS found; for final batch, try generic boundary detection - return is_final ? find_next_document_index(parser) : 0; - } - - // Phase 2: Determine batch boundaries based on RS positions - - if (is_final) { - // Final batch: all documents are complete (last one ends at EOF). - // In json_sequence mode, RS markers define document boundaries, so all - // remaining structurals form complete documents. Return them all directly. - // (Calling find_next_document_index() would fail for scalar documents.) - return parser.n_structural_indexes; - } - - // Partial batch: need to find complete documents only. - // A document starting at an RS is complete if there is another RS after it. - next_batch_start = last_rs_pos; - - if (rs_count < 2) { - // Only one RS, so we have at most one document that may be incomplete. - // We cannot confirm it is complete without another RS. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least 2 RS markers. The last complete document ends before last_rs_pos. - - // Find the structural index cutoff: keep only structurals < last_rs_pos. - // Since we already filtered RS, all remaining structurals are valid. - // We iterate backward to find the last structural before last_rs_pos. - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_rs_pos) { - keep_count = i; - break; - } - } - - // No structurals before the last RS - no complete documents - if (keep_count == 0) { return 0; } - - // All documents before the last RS are complete by definition (the next RS - // confirms their end). No need to call find_next_document_index() which - // would fail for scalar documents like `1` or `"hello"`. - return keep_count; -} - -/** - * Filter comma-delimited documents by removing root-level commas from - * structural indexes. - * - * For comma-delimited format like `{...},{...},{...}`, we need to remove - * the commas that separate documents (depth 0) while preserving commas - * inside arrays and objects (depth > 0). - * - * After filtering, the structural indexes look like whitespace-delimited - * documents, so find_next_document_index() works unchanged. - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @return The number of structural indexes to keep, - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t filter_comma_delimited( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Track depth to identify root-level commas (depth 0) - int depth = 0; - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_root_comma_pos = 0; - uint32_t root_comma_count = 0; - - for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { - uint32_t idx = parser.structural_indexes[i]; - uint8_t c = parser.buf[idx]; - - switch (c) { - case '{': case '[': - depth++; - parser.structural_indexes[write_idx++] = idx; - break; - case '}': case ']': - depth--; - parser.structural_indexes[write_idx++] = idx; - break; - case ',': - if (depth == 0) { - // Root-level comma = document boundary, skip it - last_root_comma_pos = idx; - root_comma_count++; - continue; - } - parser.structural_indexes[write_idx++] = idx; - break; - default: - // Colons, scalars, etc. - parser.structural_indexes[write_idx++] = idx; - break; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { return 0; } - - if (is_final) { - // Final batch: use standard boundary detection on filtered indexes - return find_next_document_index(parser); - } - - // Partial batch: need to find complete documents only. - // A document ending with a root comma is complete. - if (root_comma_count == 0) { - // No root commas found; we cannot confirm any document is complete. - // The whole batch might be one incomplete document. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least one root comma. Documents before the last comma are complete. - next_batch_start = last_root_comma_pos + 1; - - // Find the structural index cutoff: keep only structurals < last_root_comma_pos - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_root_comma_pos) { - keep_count = i; - break; - } - } - - if (keep_count == 0) { return 0; } - - // Use standard boundary detection on the complete portion - parser.n_structural_indexes = keep_count; - return find_next_document_index(parser); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H -/* end file generic/stage1/find_next_document_index.h for icelake */ -/* including generic/stage1/json_minifier.h for icelake: #include */ -/* begin file generic/stage1/json_minifier.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace icelake { -namespace { -namespace stage1 { - -class json_minifier { -public: - template - static error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept; - -private: - simdjson_inline json_minifier(uint8_t *_dst) - : dst{_dst} - {} - template - simdjson_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_warn_unused simdjson_inline error_code finish(uint8_t *dst_start, size_t &dst_len); - json_scanner scanner{}; - uint8_t *dst; -}; - -simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { - uint64_t mask = block.whitespace(); - dst += in.compress(mask, dst); -} - -simdjson_warn_unused simdjson_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { - error_code error = scanner.finish(); - if (error) { dst_len = 0; return error; } - dst_len = dst - dst_start; - return SUCCESS; -} - -template<> -simdjson_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - simd::simd8x64 in_2(block_buf+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1); - this->next(in_2, block_2); - reader.advance(); -} - -template<> -simdjson_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - json_block block_1 = scanner.next(in_1); - this->next(block_buf, block_1); - reader.advance(); -} - -template -error_code json_minifier::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept { - buf_block_reader reader(buf, len); - json_minifier minifier(dst); - - // Index the first n-1 blocks - while (reader.has_full_block()) { - minifier.step(reader.full_block(), reader); - } - - // Index the last (remainder) block, padded with spaces - uint8_t block[STEP_SIZE]; - size_t remaining_bytes = reader.get_remainder(block); - if (remaining_bytes > 0) { - // We do not want to write directly to the output stream. Rather, we write - // to a local buffer (for safety). - uint8_t out_block[STEP_SIZE]; - uint8_t * const guarded_dst{minifier.dst}; - minifier.dst = out_block; - minifier.step(block, reader); - size_t to_write = minifier.dst - out_block; - // In some cases, we could be enticed to consider the padded spaces - // as part of the string. This is fine as long as we do not write more - // than we consumed. - if(to_write > remaining_bytes) { to_write = remaining_bytes; } - memcpy(guarded_dst, out_block, to_write); - minifier.dst = guarded_dst + to_write; - } - return minifier.finish(dst, dst_len); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H -/* end file generic/stage1/json_minifier.h for icelake */ -/* including generic/stage1/json_structural_indexer.h for icelake: #include */ -/* begin file generic/stage1/json_structural_indexer.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace icelake { -namespace { -namespace stage1 { - -class bit_indexer { -public: - uint32_t *tail; - - simdjson_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} - -#if SIMDJSON_PREFER_REVERSE_BITS - /** - * ARM lacks a fast trailing zero instruction, but it has a fast - * bit reversal instruction and a fast leading zero instruction. - * Thus it may be profitable to reverse the bits (once) and then - * to rely on a sequence of instructions that call the leading - * zero instruction. - * - * Performance notes: - * The chosen routine is not optimal in terms of data dependency - * since zero_leading_bit might require two instructions. However, - * it tends to minimize the total number of instructions which is - * beneficial. - */ - simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { - int lz = leading_zeroes(rev_bits); - this->tail[i] = static_cast(idx) + lz; - rev_bits = zero_leading_bit(rev_bits, lz); - } -#else - /** - * Under recent x64 systems, we often have both a fast trailing zero - * instruction and a fast 'clear-lower-bit' instruction so the following - * algorithm can be competitive. - */ - - simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { - this->tail[i] = idx + trailing_zeroes(bits); - bits = clear_lowest_bit(bits); - } -#endif // SIMDJSON_PREFER_REVERSE_BITS - - template - simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { - write_index(idx, bits, START); - SIMDJSON_IF_CONSTEXPR (N > 1) { - write_indexes<(N-1>0?START+1:START), (N-1>=0?N-1:1)>(idx, bits); - } - return START+N; - } - - template - simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { - write_indexes(idx, bits); - SIMDJSON_IF_CONSTEXPR ((START+STEP) < END) { - if (simdjson_unlikely((START+STEP) < cnt)) { - write_indexes_stepped<(START+STEP(idx, bits, cnt); - } - } - return ((END-START) % STEP) == 0 ? END : (END-START) - ((END-START) % STEP) + STEP; - } - - // flatten out values in 'bits' assuming that they are are to have values of idx - // plus their position in the bitvector, and store these indexes at - // base_ptr[base] incrementing base as we go - // will potentially store extra values beyond end of valid bits, so base_ptr - // needs to be large enough to handle this - // - // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it - // will provide its own version of the code. -#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - simdjson_inline void write(uint32_t idx, uint64_t bits); -#else - simdjson_inline void write(uint32_t idx, uint64_t bits) { - // In some instances, the next branch is expensive because it is mispredicted. - // Unfortunately, in other cases, - // it helps tremendously. - if (bits == 0) - return; - - int cnt = static_cast(count_ones(bits)); - -#if SIMDJSON_PREFER_REVERSE_BITS - bits = reverse_bits(bits); -#endif -#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP - static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; -#else - static constexpr const int STEP = 4; -#endif - static constexpr const int STEP_UNTIL = 24; - - write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); - SIMDJSON_IF_CONSTEXPR (STEP_UNTIL < 64) { - if (simdjson_unlikely(STEP_UNTIL < cnt)) { - for (int i=STEP_UNTIL; itail += cnt; - } -#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -}; - -class json_structural_indexer { -public: - /** - * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. - * - * @param partial Setting the partial parameter to true allows the find_structural_bits to - * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If - * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. - */ - template - static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept; - -private: - simdjson_inline json_structural_indexer(uint32_t *structural_indexes); - template - simdjson_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_warn_unused simdjson_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); - - json_scanner scanner{}; - utf8_checker checker{}; - bit_indexer indexer; - uint64_t prev_structurals = 0; - uint64_t unescaped_chars_error = 0; -}; - -simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} - -// Skip the last character if it is partial -simdjson_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { - if (simdjson_unlikely(len < 3)) { - switch (len) { - case 2: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left - return len; - case 1: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - return len; - case 0: - return len; - } - } - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left - return len; -} - -// -// PERF NOTES: -// We pipe 2 inputs through these stages: -// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load -// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. -// 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. -// The output of step 1 depends entirely on this information. These functions don't quite use -// up enough CPU: the second half of the functions is highly serial, only using 1 execution core -// at a time. The second input's scans has some dependency on the first ones finishing it, but -// they can make a lot of progress before they need that information. -// 3. Step 1 does not use enough capacity, so we run some extra stuff while we're waiting for that -// to finish: utf-8 checks and generating the output from the last iteration. -// -// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all -// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough -// workout. -// -template -error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept { - if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } - // We guard the rest of the code so that we can assume that len > 0 throughout. - if (len == 0) { return EMPTY; } - if (is_streaming(partial)) { - len = trim_partial_utf8(buf, len); - // If you end up with an empty window after trimming - // the partial UTF-8 bytes, then chances are good that you - // have an UTF-8 formatting error. - if(len == 0) { return UTF8_ERROR; } - } - buf_block_reader reader(buf, len); - json_structural_indexer indexer(parser.structural_indexes.get()); - - // Read all but the last block - while (reader.has_full_block()) { - indexer.step(reader.full_block(), reader); - } - // Take care of the last block (will always be there unless file is empty which is - // not supposed to happen.) - uint8_t block[STEP_SIZE]; - const size_t remaining = parser._unpadded ? reader.get_remainder(block) : reader.get_remainder_padded(block); - if (simdjson_unlikely(remaining == 0)) { return UNEXPECTED_ERROR; } - indexer.step(block, reader); - return indexer.finish(parser, reader.block_index(), len, partial); -} - -template<> -simdjson_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block); - simd::simd8x64 in_2(block+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1, reader.block_index()); - this->next(in_2, block_2, reader.block_index()+64); - reader.advance(); -} - -template<> -simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block); - json_block block_1 = scanner.next(in_1); - this->next(in_1, block_1, reader.block_index()); - reader.advance(); -} - -simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { - uint64_t unescaped = in.lteq(0x1F); -#if SIMDJSON_UTF8VALIDATION - checker.check_next_input(in); -#endif - indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser - prev_structurals = block.structural_start(); - unescaped_chars_error |= block.non_quote_inside_string(unescaped); -} - -// Drops the unclosed string that ends a json_sequence batch and leaves its -// start in structural_indexes[n_structural_indexes]. The last index is its -// opening quote, unless the quote directly follows an RS: the scanner reads -// RS as a scalar byte, so it indexes the RS and not the quote. Keep the RS. -simdjson_inline void drop_json_sequence_unclosed_string(dom_parser_implementation &parser, size_t len) noexcept { - uint32_t quote = parser.structural_indexes[parser.n_structural_indexes - 1]; - if (parser.buf[quote] == '"') { - parser.n_structural_indexes--; - return; - } - while (quote < len && parser.buf[quote] != '"') { quote++; } - parser.structural_indexes[parser.n_structural_indexes] = quote; -} - -simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial) { - // Write out the final iteration's structurals - indexer.write(uint32_t(idx-64), prev_structurals); - error_code error = scanner.finish(); - // We deliberately break down the next expression so that it is - // human readable. - const bool should_we_exit = is_streaming(partial) ? - ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING - : (error != SUCCESS); // if partial is false, we must have SUCCESS - const bool have_unclosed_string = (error == UNCLOSED_STRING); - parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); - /*** - * The On-Demand API requires special padding. - * - * This is related to https://github.com/simdjson/simdjson/issues/906 - * Basically, we want to make sure that if the parsing continues beyond the last (valid) - * structural character, it quickly stops. - * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. - * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing - * continues, then it must be [,] or }. - * Suppose it is ] or }. We backtrack to the first character, what could it be that would - * not trigger an error? It could be ] or } but no, because you can't start a document that way. - * It can't be a comma, a colon or any simple value. So the only way we could continue is - * if the repeated character is [. But if so, the document must start with [. But if the document - * starts with [, it should end with ]. If we enforce that rule, then we would get - * ][[ which is invalid. - * - * This is illustrated with the test array_iterate_unclosed_error() on the following input: - * R"({ "a": [,,)" - **/ - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final - parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); - parser.structural_indexes[parser.n_structural_indexes + 2] = 0; - parser.next_structural_index = 0; - - // Bail out only once the count and sentinels above are set: - // document_stream::truncated_bytes() reads them even on error. - if (simdjson_unlikely(should_we_exit)) { return error; } - if (unescaped_chars_error) { return UNESCAPED_CHARS; } - - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - return EMPTY; - } - if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { - return UNEXPECTED_ERROR; - } - if (partial == stage1_mode::streaming_partial) { - // If we have an unclosed string, then the last structural - // will be the quote and we want to make sure to omit it. - if(have_unclosed_string) { - parser.n_structural_indexes--; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - // We truncate the input to the end of the last complete document (or zero). - const bool partial_scalar = !have_unclosed_string && ends_with_partial_scalar(parser, len); - const bool unfinished = ends_with_unfinished_document(parser); - auto new_structural_indexes = find_next_document_index(parser, partial_scalar || unfinished); - if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { - // The batch holds a single document, whose brackets balance but which - // cannot be valid JSON: it is not too big, it is invalid. - if (unfinished && find_next_document_index(parser, partial_scalar) != 0) { return TAPE_ERROR; } - if(parser.structural_indexes[0] == 0) { - // If the buffer is partial and we started at index 0 but the document is - // incomplete, it's too big to parse. - return CAPACITY; - } else { - // It is possible that the document could be parsed, we just had a lot - // of white space. - parser.n_structural_indexes = 0; - return EMPTY; - } - } - parser.n_structural_indexes = new_structural_indexes; - } else if (partial == stage1_mode::streaming_final) { - if(have_unclosed_string) { parser.n_structural_indexes--; } - // We truncate the input to the end of the last complete document (or zero). - // Because partial == stage1_mode::streaming_final, it means that we may - // silently ignore trailing garbage. Though it sounds bad, we do it - // deliberately because many people who have streams of JSON documents - // will truncate them for processing. E.g., imagine that you are uncompressing - // the data from a size file or receiving it in chunks from the network. You - // may not know where exactly the last document will be. Meanwhile the - // document_stream instances allow people to know the JSON documents they are - // parsing (see the iterator.source() method). - parser.n_structural_indexes = find_next_document_index(parser); - // We store the initial n_structural_indexes so that the client can see - // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, - // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, - // otherwise, it will copy some prior index. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - // This next line is critical, do not change it unless you understand what you are - // doing. - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - // We tolerate an unclosed string at the very end of the stream. Indeed, users - // often load their data in bulk without being careful and they want us to ignore - // the trailing garbage. - return EMPTY; - } - } else if (partial == stage1_mode::json_sequence_partial) { - // RFC 7464: use RS positions for batch boundaries - // A discarded unclosed string also caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - // The next batch starts at an RS, which the On-Demand API never reads as a - // string, a container or a separator. But an invalid last document may make - // it skip that RS as a value: repeat it, so that the index past it is not - // the next record, which can be a string that does not close in this batch. - parser.structural_indexes[parser.n_structural_indexes + 1] = next_batch_start; - } else if (partial == stage1_mode::json_sequence_final) { - // RFC 7464: final batch, last document extends to EOF - // As above: a discarded unclosed string caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); - // The filter compacted structural_indexes in place and restored the EOF - // sentinel past the compacted end, so the copy below is either the start - // of a truncated document or len, as in streaming_final. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } else if (partial == stage1_mode::comma_delimited_partial) { - // Comma-delimited: filter root-level commas, use comma positions for batch boundaries - if(have_unclosed_string) { - parser.n_structural_indexes--; - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - } else if (partial == stage1_mode::comma_delimited_final) { - // Comma-delimited: final batch, last document extends to EOF - if(have_unclosed_string) { parser.n_structural_indexes--; } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } - checker.check_eof(); - return checker.errors(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. -#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H -/* end file generic/stage1/json_structural_indexer.h for icelake */ -/* including generic/stage1/utf8_validator.h for icelake: #include */ -/* begin file generic/stage1/utf8_validator.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { -namespace stage1 { - -/** - * Validates that the string is actual UTF-8. - */ -template -bool generic_validate_utf8(const uint8_t * input, size_t length) { - checker c{}; - buf_block_reader<64> reader(input, length); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - c.check_next_input(in); - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - c.check_next_input(in); - reader.advance(); - c.check_eof(); - return c.errors() == error_code::SUCCESS; -} - -bool generic_validate_utf8(const char * input, size_t length) { - return generic_validate_utf8(reinterpret_cast(input),length); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H -/* end file generic/stage1/utf8_validator.h for icelake */ -/* end file generic/stage1/amalgamated.h for icelake */ -/* including generic/stage2/amalgamated.h for icelake: #include */ -/* begin file generic/stage2/amalgamated.h for icelake */ -// Stuff other things depend on -/* including generic/stage2/base.h for icelake: #include */ -/* begin file generic/stage2/base.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { -namespace stage2 { - -class json_iterator; -struct tape_builder; -struct tape_writer; - -} // namespace stage2 -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H -/* end file generic/stage2/base.h for icelake */ -/* including generic/stage2/tape_writer.h for icelake: #include */ -/* begin file generic/stage2/tape_writer.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace icelake { -namespace { -namespace stage2 { - -struct tape_writer { - /** The next place to write to tape */ - uint64_t *next_tape_loc; - - /** Write a signed 64-bit value to tape. */ - simdjson_inline void append_s64(int64_t value) noexcept; - - /** Write an unsigned 64-bit value to tape. */ - simdjson_inline void append_u64(uint64_t value) noexcept; - - /** Write a double value to tape. */ - simdjson_inline void append_double(double value) noexcept; - - /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ - simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; - - /** - * Append a tape entry (an 8-bit type,and 56 bits worth of value). - */ - simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; - - /** - * Skip the current tape entry without writing. - * - * Used to skip the start of the container, since we'll come back later to fill it in when the - * container ends. - */ - simdjson_inline void skip() noexcept; - - /** - * Skip the number of tape entries necessary to write a large u64 or i64. - */ - simdjson_inline void skip_large_integer() noexcept; - - /** - * Skip the number of tape entries necessary to write a double. - */ - simdjson_inline void skip_double() noexcept; - - /** - * Write a value to a known location on tape. - * - * Used to go back and write out the start of a container after the container ends. - */ - simdjson_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; - -private: - /** - * Append both the tape entry, and a supplementary value following it. Used for types that need - * all 64 bits, such as double and uint64_t. - */ - template - simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; -}; // struct tape_writer - -simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { - append2(0, value, internal::tape_type::INT64); -} - -simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { - append(0, internal::tape_type::UINT64); - *next_tape_loc = value; - next_tape_loc++; -} - -/** Write a double value to tape. */ -simdjson_inline void tape_writer::append_double(double value) noexcept { - append2(0, value, internal::tape_type::DOUBLE); -} - -simdjson_inline void tape_writer::skip() noexcept { - next_tape_loc++; -} - -simdjson_inline void tape_writer::skip_large_integer() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::skip_double() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { - *next_tape_loc = val | ((uint64_t(char(t))) << 56); - next_tape_loc++; -} - -template -simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { - append(val, t); - static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); - memcpy(next_tape_loc, &val2, sizeof(val2)); - next_tape_loc++; -} - -simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { - tape_loc = val | ((uint64_t(char(t))) << 56); -} - -simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { - // Write to string buffer: [4-byte LE length][digits][null] - uint32_t str_len = uint32_t(len); - memcpy(string_buf, &str_len, sizeof(uint32_t)); - memcpy(string_buf + sizeof(uint32_t), src, len); - string_buf[sizeof(uint32_t) + len] = 0; - // Tape entry: offset into string buffer - // The caller must set the offset relative to doc.string_buf base - append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches - string_buf += sizeof(uint32_t) + len + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H -/* end file generic/stage2/tape_writer.h for icelake */ -/* including generic/stage2/logger.h for icelake: #include */ -/* begin file generic/stage2/logger.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - - -// This is for an internal-only stage 2 specific logger. -// Set LOG_ENABLED = true to log what stage 2 is doing! -namespace simdjson { -namespace icelake { -namespace { -namespace logger { - - static constexpr const char * DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; - -#if SIMDJSON_VERBOSE_LOGGING - static constexpr const bool LOG_ENABLED = true; -#else - static constexpr const bool LOG_ENABLED = false; -#endif - static constexpr const int LOG_EVENT_LEN = 20; - static constexpr const int LOG_BUFFER_LEN = 30; - static constexpr const int LOG_SMALL_BUFFER_LEN = 10; - static constexpr const int LOG_INDEX_LEN = 5; - - static int log_depth; // Not threadsafe. Log only. - - // Helper to turn unprintable or newline characters into spaces - static simdjson_inline char printable_char(char c) { - if (c >= 0x20) { - return c; - } else { - return ' '; - } - } - - // Print the header and set up log_start - static simdjson_inline void log_start() { - if (LOG_ENABLED) { - log_depth = 0; - printf("\n"); - printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); - printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN+2, DASHES, LOG_BUFFER_LEN+2, DASHES, LOG_SMALL_BUFFER_LEN+2, DASHES, 5+2, DASHES); - } - } - - simdjson_unused static simdjson_inline void log_string(const char *message) { - if (LOG_ENABLED) { - printf("%s\n", message); - } - } - - // Logs a single line from the stage 2 DOM parser - template - static simdjson_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { - if (LOG_ENABLED) { - printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); - auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; - auto next_index = structurals.next_structural; - auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); - auto next = &structurals.buf[*next_index]; - { - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i=0;i */ -/* begin file generic/stage2/json_iterator.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { -namespace stage2 { - -class json_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - uint32_t depth{0}; - - /** - * Walk the JSON document. - * - * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as - * the first parameter; some callbacks have other parameters as well: - * - * - visit_document_start() - at the beginning. - * - visit_document_end() - at the end (if things were successful). - * - * - visit_array_start() - at the start `[` of a non-empty array. - * - visit_array_end() - at the end `]` of a non-empty array. - * - visit_empty_array() - when an empty array is encountered. - * - * - visit_object_end() - at the start `]` of a non-empty object. - * - visit_object_start() - at the end `]` of a non-empty object. - * - visit_empty_object() - when an empty object is encountered. - * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is - * guaranteed to point at the first quote of the string (`"key"`). - * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. - * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. - * - * - increment_count(iter) - each time a value is found in an array or object. - */ - template - simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; - - /** - * Create an iterator capable of walking a JSON document. - * - * The document must have already passed through stage 1. - */ - simdjson_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); - - /** - * Look at the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *peek(const uint32_t *batch_end = nullptr) const noexcept; - /** - * Advance to the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *advance(const uint32_t *batch_end = nullptr) noexcept; - /** - * Get the remaining length of the document, from the start of the current token. - */ - simdjson_inline size_t remaining_len() const noexcept; - /** - * Check if we are at the end of the document. - * - * If this is true, there are no more tokens. - */ - simdjson_inline bool at_eof() const noexcept; - /** - * Check if we are at the beginning of the document. - */ - simdjson_inline bool at_beginning() const noexcept; - simdjson_inline uint8_t last_structural() const noexcept; - - /** - * Log that a value has been found. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_value(const char *type) const noexcept; - /** - * Log the start of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_start_value(const char *type) const noexcept; - /** - * Log the end of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_end_value(const char *type) const noexcept; - /** - * Log an error. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_error(const char *error) const noexcept; - - template - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; - template - simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; -}; - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { - logger::log_start(); - - // - // Start the document - // - if (at_eof()) { return EMPTY; } - // Only used when BOUNDED (see advance()); a local, so that it can stay in a - // register, and vanishes from the other walks. - const uint32_t *const batch_end = &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - log_start_value("document"); - SIMDJSON_TRY( visitor.visit_document_start(*this) ); - - // - // Read first value - // - { - auto value = advance(batch_end); - - // Make sure the outer object or array is closed before continuing; otherwise, there are ways we - // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 - if (!STREAMING) { - switch (*value) { - case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; - case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; - } - } - - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; - } - } - goto document_end; - -// -// Object parser states -// -object_begin: - log_start_value("object"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = false; - SIMDJSON_TRY( visitor.visit_object_start(*this) ); - - { - auto key = advance(batch_end); - if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.increment_count(*this) ); - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - -object_field: - if (simdjson_unlikely( (*advance(batch_end) != ':') )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -object_continue: - switch (*advance(batch_end)) { - case ',': - SIMDJSON_TRY( visitor.increment_count(*this) ); - { - auto key = advance(batch_end); - if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - goto object_field; - case '}': log_end_value("object"); SIMDJSON_TRY( visitor.visit_object_end(*this) ); goto scope_end; - default: log_error("No comma between object fields"); return TAPE_ERROR; - } - -scope_end: - depth--; - if (depth == 0) { goto document_end; } - if (dom_parser.is_array[depth]) { goto array_continue; } - goto object_continue; - -// -// Array parser states -// -array_begin: - log_start_value("array"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = true; - SIMDJSON_TRY( visitor.visit_array_start(*this) ); - SIMDJSON_TRY( visitor.increment_count(*this) ); - -array_value: - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -array_continue: - switch (*advance(batch_end)) { - case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; - case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; - default: log_error("Missing comma between array values"); return TAPE_ERROR; - } - -document_end: - log_end_value("document"); - SIMDJSON_TRY( visitor.visit_document_end(*this) ); - - dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); - - // If we didn't make it to the end, it's an error - if ( !STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes ) { - log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); - return TAPE_ERROR; - } - - return SUCCESS; - -} // walk_document() - -simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { -} - -// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires -// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). -// -// When BOUNDED (see dom_parser_implementation::_bounded_batch), a document may -// not run past batch_end, one past the last structural index of the batch: -// past it, we return the same dummy, and walk_document fails cleanly. -template -simdjson_inline const uint8_t *json_iterator::peek(const uint32_t *batch_end) const noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -template -simdjson_inline const uint8_t *json_iterator::advance(const uint32_t *batch_end) noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural++); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -simdjson_inline size_t json_iterator::remaining_len() const noexcept { - return dom_parser.len - *(next_structural-1); -} - -simdjson_inline bool json_iterator::at_eof() const noexcept { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; -} -simdjson_inline bool json_iterator::at_beginning() const noexcept { - return next_structural == dom_parser.structural_indexes.get(); -} -simdjson_inline uint8_t json_iterator::last_structural() const noexcept { - return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; -} - -simdjson_inline void json_iterator::log_value(const char *type) const noexcept { - logger::log_line(*this, "", type, ""); -} - -simdjson_inline void json_iterator::log_start_value(const char *type) const noexcept { - logger::log_line(*this, "+", type, ""); - if (logger::LOG_ENABLED) { logger::log_depth++; } -} - -simdjson_inline void json_iterator::log_end_value(const char *type) const noexcept { - if (logger::LOG_ENABLED) { logger::log_depth--; } - logger::log_line(*this, "-", type, ""); -} - -simdjson_inline void json_iterator::log_error(const char *error) const noexcept { - logger::log_line(*this, "", "ERROR", error); -} - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { - switch (*value) { - case '"': return visitor.visit_root_string(*this, value); - case 't': return visitor.visit_root_true_atom(*this, value); - case 'f': return visitor.visit_root_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_root_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_root_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_root_inf_atom(*this, value); -#else - case 'n': return visitor.visit_root_null_atom(*this, value); -#endif - case '-': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return visitor.visit_root_number(*this, value); - default: - log_error("Document starts with a non-value character"); - return TAPE_ERROR; - } -} -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { - // Use the fact that most scalars are going to be either strings or numbers. - if(*value == '"') { - return visitor.visit_string(*this, value); - } else if (((*value - '0') < 10) || (*value == '-')) { - return visitor.visit_number(*this, value); - } - // true, false, null are uncommon. - switch (*value) { - case 't': return visitor.visit_true_atom(*this, value); - case 'f': return visitor.visit_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_inf_atom(*this, value); -#else - case 'n': return visitor.visit_null_atom(*this, value); -#endif - default: - log_error("Non-value found when value was expected!"); - return TAPE_ERROR; - } -} - -} // namespace stage2 -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H -/* end file generic/stage2/json_iterator.h for icelake */ -/* including generic/stage2/stringparsing.h for icelake: #include */ -/* begin file generic/stage2/stringparsing.h for icelake */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace icelake { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - // Decode adjacent Unicode escapes without returning to the - // quote-and-backslash scanner between code points. - do { - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } while (src[0] == '\\' && src[1] == 'u'); - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -/** - * Bounds-safe variant of parse_string for input buffers that are NOT padded to - * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input - * byte (buf + len). It behaves exactly like parse_string while we are at least - * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays - * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few - * remaining bytes into a space-padded scratch buffer and finishes with the - * regular parse_string. This keeps the delicate escape/Unicode handling in one - * place (the proven parse_string) rather than duplicating it. - * - * Correctness relies on stage 1 having validated the string, i.e. there is an - * unescaped closing quote within [src, buf_end); that quote is therefore inside - * the copied scratch, so parse_string finds it without running off the scratch. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { - // Far from the end: identical to parse_string's loop. The guard uses - // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past - // buf_end; escape/Unicode look-aheads read within the string (before the - // closing quote, which is < buf_end), so they are in bounds here too. - // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: - // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate - // following bytes (+6 from the '\'), then (if it sees \u) another - // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the - // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the - // chunk start. The +12 margin ensures that even on kernels where - // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays - // in-bounds. The scratch fallback (3*PAD) is already safe. - while (src + SIMDJSON_PADDING + 12 <= buf_end) { - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - if (bs_quote.has_quote_first()) { - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - if (escape_char == 'u') { - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - // Within the final SIMDJSON_PADDING bytes: copy what remains into a - // space-padded scratch (spaces are neither quote nor backslash, so they do not - // disturb matching) and let the regular parser finish from there. The closing - // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in - // the chunk starting at some offset <= remaining and reads at most - // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. - // We size at 3x for a comfortable margin (the unicode look-ahead reads a few - // extra bytes past an escape). - uint8_t scratch[SIMDJSON_PADDING * 3]; - const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING - std::memset(scratch, ' ', sizeof(scratch)); - std::memcpy(scratch, src, remaining); - return parse_string(scratch, dst, allow_replacement); -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for icelake */ -/* including generic/stage2/tape_builder.h for icelake: #include */ -/* begin file generic/stage2/tape_builder.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - - -namespace simdjson { -namespace icelake { -namespace { -namespace stage2 { - -template -struct tape_builder_impl { - /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; - /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator &iter) noexcept; - - /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator &iter) noexcept; - /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator &iter) noexcept; - /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator &iter) noexcept; - - /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator &iter) noexcept; - /** - * Called when a key in a field is encountered. - * - * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array - * will be called after this with the field value. - */ - simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; - /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator &iter) noexcept; - /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator &iter) noexcept; - - /** - * Called when a string, number, boolean or null is found. - */ - simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; - /** - * Called when a string, number, boolean or null is found at the top level of a document (i.e. - * when there is no array or object and the entire document is a single string, number, boolean or - * null. - * - * This is separate from primitive() because simdjson's normal primitive parsing routines assume - * there is at least one more token after the value, which is only true in an array or object. - */ - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - -#if SIMDJSON_ENABLE_NAN_INF - simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, - // this returns a tape error on failure. - simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; -#endif - - /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; - - /** Next location to write to tape */ - tape_writer tape; -public: - simdjson_inline tape_builder_impl(dom::document &doc) noexcept; -private: - /** Next write location in the string buf for stage 2 parsing */ - uint8_t *current_string_buf_loc; - - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - simdjson_inline void start_container(json_iterator &iter) noexcept; - simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder_impl - -// Thin, non-templated entry so each architecture's stage2() keeps calling -// tape_builder::parse_document unchanged. It chooses the bounds-safe -// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the -// choice is a compile-time constant inside the walk: the padded path carries no -// extra branch or load (see tape_builder_impl::visit_string). -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } - - // The stage 2 entry points of dom::document_stream, which picks one per batch - // (see internal::dom_parser_implementation::stage2_next_function). Out of - // line, so that stage2_next() can share the code. - simdjson_warn_unused static simdjson_never_inline error_code parse_next_document( - internal::dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - return parse_document(static_cast(dom_parser), doc); - } - - // A document of a stream batch that stage 2 must bound to its structural - // indexes (see internal::dom_parser_implementation::_bounded_batch). - simdjson_warn_unused static simdjson_never_inline error_code parse_next_bounded_document( - internal::dom_parser_implementation &_dom_parser, dom::document &doc) noexcept { - dom_parser_implementation &dom_parser = static_cast(_dom_parser); - dom_parser.doc = &doc; - json_iterator iter(dom_parser, dom_parser.next_structural_index); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } -}; - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_root_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { - constexpr uint32_t start_tape_index = 0; - tape.append(start_tape_index, internal::tape_type::ROOT); - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { - return visit_string(iter, key, true); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 - return SUCCESS; -} - -template -simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { - iter.log_value(key ? "key" : "string"); - uint8_t *dst = on_start_string(iter); - // We do not allow replacement when the escape characters are invalid. - // UNPADDED is a compile-time constant chosen once per document by - // tape_builder::parse_document, so the padded build instantiates only the - // plain parse_string call below -- no runtime branch and no flag load. - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); - } else { - dst = stringparsing::parse_string(value+1, dst, false); - } - if (dst == nullptr) { - iter.log_error("Invalid escape in string"); - return STRING_ERROR; - } - on_end_string(dst); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { - return visit_string(iter, value); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("number"); - const uint8_t *num = value; - std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - // numberparsing reads ahead in 8-byte blocks for floats - // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a - // number whose digits reach the final bytes of an unpadded buffer would read - // past it. *(next_structural) is the offset of the token following this - // number, hence an upper bound on where the digits end; when that is within - // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail - // (mirroring visit_root_number). This fires only for numbers near the end. - if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { - const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document - copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, rl); - std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); - num = copy.get(); - } - } - error_code err = numberparsing::parse_number(num, tape); - if (simdjson_unlikely(err == BIGINT_ERROR && - iter.dom_parser._number_as_string)) { - // Write big integer to string buffer using the same format as strings. - // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = num; - if (*p == '-') p++; - while (numberparsing::is_digit(*p)) p++; - // The digit run must be terminated by a structural or whitespace character; otherwise the - // token is malformed (e.g. "123456789123456789123x"). - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - num); - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); - uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, num, len); - dst += len; - on_end_string(dst); - return SUCCESS; - } - return err; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { - // - // We need to make a copy to make sure that the string is space terminated. - // This is not about padding the input, which should already padded up - // to len + SIMDJSON_PADDING. However, we have no control at this stage - // on how the padding was done. What if the input string was padded with nulls? - // It is quite common for an input string to have an extra null character (C string). - // We do not want to allow 9\0 (where \0 is the null character) inside a JSON - // document, but the string "9\0" by itself is fine. So we make a copy and - // pad the input with spaces when we know that there is just one input element. - // This copy is relatively expensive, but it will almost never be called in - // practice unless you are in the strange scenario where you have many JSON - // documents made of single atoms. - // - // In a stream, the input goes on with other documents: copy up to the next - // structural only, not to the end of the batch. - const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); - std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, len); - std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); - error_code error = visit_number(iter, copy.get()); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated - // token at the very end of an unpadded buffer would over-read. Use the - // length-aware form there (the root variant already does this). - const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) - : atomparsing::is_valid_true_atom(value); - if (!ok) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) - : atomparsing::is_valid_false_atom(value); - if (!ok) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) - : atomparsing::is_valid_null_atom(value); - if (!ok) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -#if SIMDJSON_ENABLE_NAN_INF -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - // For unpadded input use the length-aware validator so the 'infinity'-style - // 8-byte compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) - : atomparsing::is_valid_nan_atom(value); - if (!ok) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. - // For unpadded input use the length-aware validator so the 'infinity' 8-byte - // compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) - : atomparsing::is_valid_inf_atom(value); - if (!ok) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure - if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -// private: - -template -simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { - return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - auto start_index = next_tape_index(iter); - tape.append(start_index+2, start); - tape.append(start_index, end); - return SUCCESS; -} - -template -simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); - iter.dom_parser.open_containers[iter.depth].count = 0; - tape.skip(); // We don't actually *write* the start element until the end. -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - // Write the ending tape element, pointing at the start location - const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; - tape.append(start_tape_index, end); - // Write the start tape element, pointing at the end location (and including count) - // count can overflow if it exceeds 24 bits... so we saturate - // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). - const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; - const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); - return SUCCESS; -} - -template -simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { - // we advance the point, accounting for the fact that we have a NULL termination - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); - return current_string_buf_loc + sizeof(uint32_t); -} - -template -simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { - uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); - // TODO check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); - // NULL termination is still handy if you expect all your strings to - // be NULL terminated? It comes at a small cost - *dst = 0; - current_string_buf_loc = dst + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H -/* end file generic/stage2/tape_builder.h for icelake */ -/* end file generic/stage2/amalgamated.h for icelake */ - -#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -// -// Stage 1 -// - -namespace simdjson { -namespace icelake { - -simdjson_warn_unused error_code implementation::create_dom_parser_implementation( - size_t capacity, - size_t max_depth, - std::unique_ptr& dst -) const noexcept { - dst.reset( new (std::nothrow) dom_parser_implementation() ); - if (!dst) { return MEMALLOC; } - if (auto err = dst->set_capacity(capacity)) - return err; - if (auto err = dst->set_max_depth(max_depth)) - return err; - return SUCCESS; -} - -namespace { - -using namespace simd; - -// This identifies structural characters (comma, colon, braces, brackets), -// and ASCII white-space ('\r','\n','\t',' '). -simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why - // we can't use the generic lookup_16. - const auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); - - // The 6 operators (:,[]{}) have these values: - // - // , 2C - // : 3A - // [ 5B - // { 7B - // ] 5D - // } 7D - // - // If you use | 0x20 to turn [ and ] into { and }, the lower 4 bits of each character is unique. - // We exploit this, using a simd 4-bit lookup to tell us which character match against, and then - // match it (against | 0x20). - // - // To prevent recognizing other characters, everything else gets compared with 0, which cannot - // match due to the | 0x20. - // - // NOTE: Due to the | 0x20, this ALSO treats and (control characters 0C and 1A) like , - // and :. This gets caught in stage 2, which checks the actual character to ensure the right - // operators are in the right places. - const auto op_table = simd8::repeat_16( - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, ':', '{', // : = 3A, [ = 5B, { = 7B - ',', '}', 0, 0 // , = 2C, ] = 5D, } = 7D - ); - - // We compute whitespace and op separately. If later code only uses one or the - // other, given the fact that all functions are aggressively inlined, we can - // hope that useless computations will be omitted. This is namely case when - // minifying (we only need whitespace). - - const uint64_t whitespace = in.eq({ - _mm512_shuffle_epi8(whitespace_table, in.chunks[0]) - }); - // Turn [ and ] into { and } - const simd8x64 curlified{ - in.chunks[0] | 0x20 - }; - const uint64_t op = curlified.eq({ - _mm512_shuffle_epi8(op_table, in.chunks[0]) - }); - - return { whitespace, op }; -} - -simdjson_inline bool is_ascii(const simd8x64& input) { - return input.reduce_or().is_ascii(); -} - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be >= 0x80 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be >= 0x80 - return is_third_byte | is_fourth_byte; -} - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -/** - * We provide a custom version of bit_indexer::write using - * naked intrinsics. - * TODO: make this code more elegant. - */ -// Under GCC 12, the intrinsic _mm512_extracti32x4_epi32 may generate 'maybe uninitialized'. -// as a workaround, we disable warnings within the following function. -SIMDJSON_PUSH_DISABLE_ALL_WARNINGS -namespace simdjson { namespace icelake { namespace { namespace stage1 { -simdjson_inline void bit_indexer::write(uint32_t idx, uint64_t bits) { - // In some instances, the next branch is expensive because it is mispredicted. - // Unfortunately, in other cases, - // it helps tremendously. - if (bits == 0) { return; } - - const __m512i indexes = _mm512_maskz_compress_epi8(bits, _mm512_set_epi32( - 0x3f3e3d3c, 0x3b3a3938, 0x37363534, 0x33323130, - 0x2f2e2d2c, 0x2b2a2928, 0x27262524, 0x23222120, - 0x1f1e1d1c, 0x1b1a1918, 0x17161514, 0x13121110, - 0x0f0e0d0c, 0x0b0a0908, 0x07060504, 0x03020100 - )); - const __m512i start_index = _mm512_set1_epi32(idx); - - const auto count = count_ones(bits); - __m512i t0 = _mm512_cvtepu8_epi32(_mm512_castsi512_si128(indexes)); - _mm512_storeu_si512(this->tail, _mm512_add_epi32(t0, start_index)); - - if(count > 16) { - const __m512i t1 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 1)); - _mm512_storeu_si512(this->tail + 16, _mm512_add_epi32(t1, start_index)); - if(count > 32) { - const __m512i t2 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 2)); - _mm512_storeu_si512(this->tail + 32, _mm512_add_epi32(t2, start_index)); - if(count > 48) { - const __m512i t3 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 3)); - _mm512_storeu_si512(this->tail + 48, _mm512_add_epi32(t3, start_index)); - } - } - } - this->tail += count; -} -}}}} -SIMDJSON_POP_DISABLE_WARNINGS - -// -// Stage 2 -// - -// -// Implementation-specific overrides -// -namespace simdjson { -namespace icelake { - -simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { - return icelake::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, stage1_mode streaming) noexcept { - this->buf = _buf; - this->len = _len; - return icelake::stage1::json_structural_indexer::index<128>(_buf, _len, *this, streaming); -} - -simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return icelake::stage1::generic_validate_utf8(buf,len); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_next_document(*this, _doc); -} - -internal::dom_parser_implementation::stage2_next_function_t dom_parser_implementation::stage2_next_function(bool bounded) const noexcept { - return bounded ? stage2::tape_builder::parse_next_bounded_document : stage2::tape_builder::parse_next_document; -} - -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { - return icelake::stringparsing::parse_string(src, dst, replacement_char); -} - -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { - return icelake::stringparsing::parse_wobbly_string(src, dst); -} - -simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { - auto error = stage1(_buf, _len, stage1_mode::regular); - if (error) { return error; } - return stage2(_doc); -} - -} // namespace icelake -} // namespace simdjson - -/* including simdjson/icelake/end.h: #include */ -/* begin file simdjson/icelake/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_ICELAKE -SIMDJSON_UNTARGET_REGION -#endif - -/* undefining SIMDJSON_IMPLEMENTATION from "icelake" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/icelake/end.h */ - -#endif // SIMDJSON_SRC_ICELAKE_CPP -/* end file icelake.cpp */ -#endif -#if SIMDJSON_IMPLEMENTATION_PPC64 -/* including ppc64.cpp: #include */ -/* begin file ppc64.cpp */ -#ifndef SIMDJSON_SRC_PPC64_CPP -#define SIMDJSON_SRC_PPC64_CPP - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* including simdjson/ppc64.h: #include */ -/* begin file simdjson/ppc64.h */ -#ifndef SIMDJSON_PPC64_H -#define SIMDJSON_PPC64_H - -/* including simdjson/ppc64/begin.h: #include "simdjson/ppc64/begin.h" */ -/* begin file simdjson/ppc64/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "ppc64" */ -#define SIMDJSON_IMPLEMENTATION ppc64 -/* including simdjson/ppc64/base.h: #include "simdjson/ppc64/base.h" */ -/* begin file simdjson/ppc64/base.h */ -#ifndef SIMDJSON_PPC64_BASE_H -#define SIMDJSON_PPC64_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * Implementation for ALTIVEC (PPC64). - */ -namespace ppc64 { - -class implementation; - -namespace { -namespace simd { -template struct simd8; -template struct simd8x64; -} // namespace simd -} // unnamed namespace - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_BASE_H -/* end file simdjson/ppc64/base.h */ -/* including simdjson/ppc64/intrinsics.h: #include "simdjson/ppc64/intrinsics.h" */ -/* begin file simdjson/ppc64/intrinsics.h */ -#ifndef SIMDJSON_PPC64_INTRINSICS_H -#define SIMDJSON_PPC64_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This should be the correct header whether -// you use visual studio or other compilers. -#include - -// These are defined by altivec.h in GCC toolchain, it is safe to undef them. -#ifdef bool -#undef bool -#endif - -#ifdef vector -#undef vector -#endif - -static_assert(sizeof(__vector unsigned char) <= simdjson::SIMDJSON_PADDING, "insufficient padding for ppc64"); - -#endif // SIMDJSON_PPC64_INTRINSICS_H -/* end file simdjson/ppc64/intrinsics.h */ -/* including simdjson/ppc64/bitmanipulation.h: #include "simdjson/ppc64/bitmanipulation.h" */ -/* begin file simdjson/ppc64/bitmanipulation.h */ -#ifndef SIMDJSON_PPC64_BITMANIPULATION_H -#define SIMDJSON_PPC64_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num - 1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_inline int count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num); // Visual Studio wants two underscores -} -#else -simdjson_inline int count_ones(uint64_t input_num) { - return __builtin_popcountll(input_num); -} -#endif - - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_BITMANIPULATION_H -/* end file simdjson/ppc64/bitmanipulation.h */ -/* including simdjson/ppc64/bitmask.h: #include "simdjson/ppc64/bitmask.h" */ -/* begin file simdjson/ppc64/bitmask.h */ -#ifndef SIMDJSON_PPC64_BITMASK_H -#define SIMDJSON_PPC64_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is -// encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { - // You can use the version below, however gcc sometimes miscompiles - // vec_pmsum_be, it happens somewhere around between 8 and 9th version. - // The performance boost was not noticeable, falling back to a usual - // implementation. - // __vector unsigned long long all_ones = {~0ull, ~0ull}; - // __vector unsigned long long mask = {bitmask, 0}; - // // Clang and GCC return different values for pmsum for ull so cast it to one. - // // Generally it is not specified by ALTIVEC ISA what is returned by - // // vec_pmsum_be. - // #if defined(__LITTLE_ENDIAN__) - // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[0]); - // #else - // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[1]); - // #endif - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; - return bitmask; -} - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif -/* end file simdjson/ppc64/bitmask.h */ -/* including simdjson/ppc64/numberparsing_defs.h: #include "simdjson/ppc64/numberparsing_defs.h" */ -/* begin file simdjson/ppc64/numberparsing_defs.h */ -#ifndef SIMDJSON_PPC64_NUMBERPARSING_DEFS_H -#define SIMDJSON_PPC64_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#if defined(__linux__) -#include -#elif defined(__FreeBSD__) -#include -#endif - -namespace simdjson { -namespace ppc64 { -namespace numberparsing { - -// we don't have appropriate instructions, so let us use a scalar function -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - uint64_t val; - std::memcpy(&val, chars, sizeof(uint64_t)); -#ifdef __BIG_ENDIAN__ -#if defined(__linux__) - val = bswap_64(val); -#elif defined(__FreeBSD__) - val = bswap64(val); -#endif -#endif - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace ppc64 -} // namespace simdjson - -#ifndef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_IS_BIG_ENDIAN -#define SIMDJSON_SWAR_NUMBER_PARSING 0 -#else -#define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif -#endif - -#endif // SIMDJSON_PPC64_NUMBERPARSING_DEFS_H -/* end file simdjson/ppc64/numberparsing_defs.h */ -/* including simdjson/ppc64/simd.h: #include "simdjson/ppc64/simd.h" */ -/* begin file simdjson/ppc64/simd.h */ -#ifndef SIMDJSON_PPC64_SIMD_H -#define SIMDJSON_PPC64_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace ppc64 { -namespace { -namespace simd { - -using __m128i = __vector unsigned char; - -template struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{__m128i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i &() const { - return this->value; - } - simdjson_inline operator __m128i &() { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { - return vec_or(this->value, (__m128i)other); - } - simdjson_inline Child operator&(const Child other) const { - return vec_and(this->value, (__m128i)other); - } - simdjson_inline Child operator^(const Child other) const { - return vec_xor(this->value, (__m128i)other); - } - simdjson_inline Child bit_andnot(const Child other) const { - return vec_andc(this->value, (__m128i)other); - } - simdjson_inline Child &operator|=(const Child other) { - auto this_cast = static_cast(this); - *this_cast = *this_cast | other; - return *this_cast; - } - simdjson_inline Child &operator&=(const Child other) { - auto this_cast = static_cast(this); - *this_cast = *this_cast & other; - return *this_cast; - } - simdjson_inline Child &operator^=(const Child other) { - auto this_cast = static_cast(this); - *this_cast = *this_cast ^ other; - return *this_cast; - } -}; - -template > -struct base8 : base> { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { - return (__m128i)vec_cmpeq(lhs.value, (__m128i)rhs); - } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(simd8 prev_chunk) const { - __m128i chunk = this->value; -#ifdef __LITTLE_ENDIAN__ - chunk = (__m128i)vec_reve(this->value); - prev_chunk = (__m128i)vec_reve((__m128i)prev_chunk); -#endif - chunk = (__m128i)vec_sld((__m128i)prev_chunk, (__m128i)chunk, 16 - N); -#ifdef __LITTLE_ENDIAN__ - chunk = (__m128i)vec_reve((__m128i)chunk); -#endif - return chunk; - } -}; - -// SIMD byte mask type (returned by things like eq and gt) -template <> struct simd8 : base8 { - static simdjson_inline simd8 splat(bool _value) { - return (__m128i)vec_splats((unsigned char)(-(!!_value))); - } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) - : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) - : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { - __vector unsigned long long result; - const __m128i perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40, - 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00}; - - result = ((__vector unsigned long long)vec_vbpermq((__m128i)this->value, - (__m128i)perm_mask)); -#ifdef __LITTLE_ENDIAN__ - return static_cast(result[1]); -#else - return static_cast(result[0]); -#endif - } - simdjson_inline bool any() const { - return !vec_all_eq(this->value, (__m128i)vec_splats(0)); - } - simdjson_inline simd8 operator~() const { - return this->value ^ (__m128i)splat(true); - } -}; - -template struct base8_numeric : base8 { - static simdjson_inline simd8 splat(T value) { - (void)value; - return (__m128i)vec_splats(value); - } - static simdjson_inline simd8 zero() { return splat(0); } - static simdjson_inline simd8 load(const T values[16]) { - return (__m128i)(vec_vsx_ld(0, reinterpret_cast(values))); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, - T v5, T v6, T v7, T v8, T v9, - T v10, T v11, T v12, T v13, - T v14, T v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, - v14, v15); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) - : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { - vec_vsx_st(this->value, 0, reinterpret_cast<__m128i *>(dst)); - } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { - return (__m128i)((__m128i)this->value + (__m128i)other); - } - simdjson_inline simd8 operator-(const simd8 other) const { - return (__m128i)((__m128i)this->value - (__m128i)other); - } - simdjson_inline simd8 &operator+=(const simd8 other) { - *this = *this + other; - return *static_cast *>(this); - } - simdjson_inline simd8 &operator-=(const simd8 other) { - *this = *this - other; - return *static_cast *>(this); - } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior - // for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return (__m128i)vec_perm((__m128i)lookup_table, (__m128i)lookup_table, this->value); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted - // as a bitset). Passing a 0 value for mask would be equivalent to writing out - // every byte to output. Only the first 16 - count_ones(mask) bytes of the - // result are significant but 16 bytes get written. Design consideration: it - // seems like a function with the signature simd8 compress(uint32_t mask) - // would be sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L *output) const { - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - using internal::thintable_epi8; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. -#ifdef __LITTLE_ENDIAN__ - __m128i shufmask = (__m128i)(__vector unsigned long long){ - thintable_epi8[mask1], thintable_epi8[mask2]}; -#else - __m128i shufmask = (__m128i)(__vector unsigned long long){ - thintable_epi8[mask2], thintable_epi8[mask1]}; - shufmask = (__m128i)vec_reve((__m128i)shufmask); -#endif - // we increment by 0x08 the second half of the mask - shufmask = ((__m128i)shufmask) + - ((__m128i)(__vector int){0, 0, 0x08080808, 0x08080808}); - - // this is the version "nearly pruned" - __m128i pruned = vec_perm(this->value, this->value, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - __m128i compactmask = - vec_vsx_ld(0, reinterpret_cast(pshufb_combine_table + pop1 * 8)); - __m128i answer = vec_perm(pruned, (__m128i)vec_splats(0), compactmask); - vec_vsx_st(answer, 0, reinterpret_cast<__m128i *>(output)); - } - - template - simdjson_inline simd8 - lookup_16(L replace0, L replace1, L replace2, L replace3, L replace4, - L replace5, L replace6, L replace7, L replace8, L replace9, - L replace10, L replace11, L replace12, L replace13, L replace14, - L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, replace4, replace5, replace6, - replace7, replace8, replace9, replace10, replace11, replace12, - replace13, replace14, replace15)); - } -}; - -// Signed bytes -template <> struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) - : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t *values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3, - int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, - int8_t v12, int8_t v13, int8_t v14, int8_t v15) - : simd8((__m128i)(__vector signed char){v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, - v15}) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 - repeat_16(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, - int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, - int8_t v12, int8_t v13, int8_t v14, int8_t v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15); - } - - // Order-sensitive comparisons - simdjson_inline simd8 - max_val(const simd8 other) const { - return (__m128i)vec_max((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - simdjson_inline simd8 - min_val(const simd8 other) const { - return (__m128i)vec_min((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - simdjson_inline simd8 - operator>(const simd8 other) const { - return (__m128i)vec_cmpgt((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - simdjson_inline simd8 - operator<(const simd8 other) const { - return (__m128i)vec_cmplt((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } -}; - -// Unsigned bytes -template <> struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) - : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t *values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline - simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, - uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, - uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) - : simd8((__m128i){v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15}) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 - repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, - uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, - uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, - uint8_t v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15); - } - - // Saturated math - simdjson_inline simd8 - saturating_add(const simd8 other) const { - return (__m128i)vec_adds(this->value, (__m128i)other); - } - simdjson_inline simd8 - saturating_sub(const simd8 other) const { - return (__m128i)vec_subs(this->value, (__m128i)other); - } - - // Order-specific operations - simdjson_inline simd8 - max_val(const simd8 other) const { - return (__m128i)vec_max(this->value, (__m128i)other); - } - simdjson_inline simd8 - min_val(const simd8 other) const { - return (__m128i)vec_min(this->value, (__m128i)other); - } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 - gt_bits(const simd8 other) const { - return this->saturating_sub(other); - } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 - lt_bits(const simd8 other) const { - return other.saturating_sub(*this); - } - simdjson_inline simd8 - operator<=(const simd8 other) const { - return other.max_val(*this) == other; - } - simdjson_inline simd8 - operator>=(const simd8 other) const { - return other.min_val(*this) == other; - } - simdjson_inline simd8 - operator>(const simd8 other) const { - return this->gt_bits(other).any_bits_set(); - } - simdjson_inline simd8 - operator<(const simd8 other) const { - return this->gt_bits(other).any_bits_set(); - } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { - return (__m128i)vec_cmpeq(this->value, (__m128i)vec_splats(uint8_t(0))); - } - simdjson_inline simd8 bits_not_set(simd8 bits) const { - return (*this & bits).bits_not_set(); - } - simdjson_inline simd8 any_bits_set() const { - return ~this->bits_not_set(); - } - simdjson_inline simd8 any_bits_set(simd8 bits) const { - return ~this->bits_not_set(bits); - } - simdjson_inline bool bits_not_set_anywhere() const { - return vec_all_eq(this->value, (__m128i)vec_splats(0)); - } - simdjson_inline bool any_bits_set_anywhere() const { - return !bits_not_set_anywhere(); - } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { - return vec_all_eq(vec_and(this->value, (__m128i)bits), - (__m128i)vec_splats(0)); - } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { - return !bits_not_set_anywhere(bits); - } - template simdjson_inline simd8 shr() const { - return simd8( - (__m128i)vec_sr(this->value, (__m128i)vec_splat_u8(N))); - } - template simdjson_inline simd8 shl() const { - return simd8( - (__m128i)vec_sl(this->value, (__m128i)vec_splat_u8(N))); - } -}; - -template struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, - "PPC64 kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64 &o) = delete; // no copy allowed - simd8x64 & - operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, - const simd8 chunk2, const simd8 chunk3) - : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_inline simd8x64(const T ptr[64]) - : chunks{simd8::load(ptr), simd8::load(ptr + 16), - simd8::load(ptr + 32), simd8::load(ptr + 48)} {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr + sizeof(simd8) * 0); - this->chunks[1].store(ptr + sizeof(simd8) * 1); - this->chunks[2].store(ptr + sizeof(simd8) * 2); - this->chunks[3].store(ptr + sizeof(simd8) * 3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | - (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t compress(uint64_t mask, T *output) const { - this->chunks[0].compress(uint16_t(mask), output); - this->chunks[1].compress(uint16_t(mask >> 16), - output + 16 - count_ones(mask & 0xFFFF)); - this->chunks[2].compress(uint16_t(mask >> 32), - output + 32 - count_ones(mask & 0xFFFFFFFF)); - this->chunks[3].compress(uint16_t(mask >> 48), - output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); - return 64 - count_ones(mask); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r1 = this->chunks[1].to_bitmask(); - uint64_t r2 = this->chunks[2].to_bitmask(); - uint64_t r3 = this->chunks[3].to_bitmask(); - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] == mask, this->chunks[1] == mask, - this->chunks[2] == mask, this->chunks[3] == mask) - .to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64(this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3]) - .to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] <= mask, this->chunks[1] <= mask, - this->chunks[2] <= mask, this->chunks[3] <= mask) - .to_bitmask(); - } -}; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_SIMD_INPUT_H -/* end file simdjson/ppc64/simd.h */ -/* including simdjson/ppc64/stringparsing_defs.h: #include "simdjson/ppc64/stringparsing_defs.h" */ -/* begin file simdjson/ppc64/stringparsing_defs.h */ -#ifndef SIMDJSON_PPC64_STRINGPARSING_DEFS_H -#define SIMDJSON_PPC64_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/simd.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote - copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { - return ((bs_bits - 1) & quote_bits) != 0; - } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { - return trailing_zeroes(quote_bits); - } - simdjson_inline int backslash_index() { - return trailing_zeroes(bs_bits); - } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote -backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), - "backslash and quote finder must process fewer than " - "SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + sizeof(v0)); - v0.store(dst); - v1.store(dst + sizeof(v0)); - - // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on - // PPC; therefore, we smash them together into a 64-byte mask and get the - // bitmask from there. - uint64_t bs_and_quote = - simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; -} - - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_STRINGPARSING_DEFS_H -/* end file simdjson/ppc64/stringparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 -/* end file simdjson/ppc64/begin.h */ -/* including simdjson/generic/amalgamated.h for ppc64: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for ppc64 */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for ppc64: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for ppc64 */ -/* including simdjson/generic/jsoncharutils.h for ppc64: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for ppc64 */ -/* including simdjson/generic/atomparsing.h for ppc64: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace ppc64 { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for ppc64 */ -/* including simdjson/generic/dom_parser_implementation.h for ppc64: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace ppc64 -} // namespace simdjson - -namespace simdjson { -namespace ppc64 { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for ppc64 */ -/* including simdjson/generic/implementation_simdjson_result_base.h for ppc64: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for ppc64 */ -/* including simdjson/generic/numberparsing.h for ppc64: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace ppc64 { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for ppc64 */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for ppc64: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for ppc64 */ -/* end file simdjson/generic/amalgamated.h for ppc64 */ -/* including simdjson/ppc64/end.h: #include "simdjson/ppc64/end.h" */ -/* begin file simdjson/ppc64/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT -/* undefining SIMDJSON_IMPLEMENTATION from "ppc64" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/ppc64/end.h */ - -#endif // SIMDJSON_PPC64_H -/* end file simdjson/ppc64.h */ -/* including simdjson/ppc64/implementation.h: #include */ -/* begin file simdjson/ppc64/implementation.h */ -#ifndef SIMDJSON_PPC64_IMPLEMENTATION_H -#define SIMDJSON_PPC64_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - -/** - * Implementation for ALTIVEC (PPC64). - */ -namespace ppc64 { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() - : simdjson::implementation("ppc64", "PPC64 ALTIVEC", - internal::instruction_set::ALTIVEC) {} - - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, size_t max_length, - std::unique_ptr &dst) - const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, - uint8_t *dst, - size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, - size_t len) const noexcept final; -}; - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_IMPLEMENTATION_H -/* end file simdjson/ppc64/implementation.h */ - -/* including simdjson/ppc64/begin.h: #include */ -/* begin file simdjson/ppc64/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "ppc64" */ -#define SIMDJSON_IMPLEMENTATION ppc64 -/* including simdjson/ppc64/base.h: #include "simdjson/ppc64/base.h" */ -/* begin file simdjson/ppc64/base.h */ -#ifndef SIMDJSON_PPC64_BASE_H -#define SIMDJSON_PPC64_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * Implementation for ALTIVEC (PPC64). - */ -namespace ppc64 { - -class implementation; - -namespace { -namespace simd { -template struct simd8; -template struct simd8x64; -} // namespace simd -} // unnamed namespace - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_BASE_H -/* end file simdjson/ppc64/base.h */ -/* including simdjson/ppc64/intrinsics.h: #include "simdjson/ppc64/intrinsics.h" */ -/* begin file simdjson/ppc64/intrinsics.h */ -#ifndef SIMDJSON_PPC64_INTRINSICS_H -#define SIMDJSON_PPC64_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This should be the correct header whether -// you use visual studio or other compilers. -#include - -// These are defined by altivec.h in GCC toolchain, it is safe to undef them. -#ifdef bool -#undef bool -#endif - -#ifdef vector -#undef vector -#endif - -static_assert(sizeof(__vector unsigned char) <= simdjson::SIMDJSON_PADDING, "insufficient padding for ppc64"); - -#endif // SIMDJSON_PPC64_INTRINSICS_H -/* end file simdjson/ppc64/intrinsics.h */ -/* including simdjson/ppc64/bitmanipulation.h: #include "simdjson/ppc64/bitmanipulation.h" */ -/* begin file simdjson/ppc64/bitmanipulation.h */ -#ifndef SIMDJSON_PPC64_BITMANIPULATION_H -#define SIMDJSON_PPC64_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num - 1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_inline int count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num); // Visual Studio wants two underscores -} -#else -simdjson_inline int count_ones(uint64_t input_num) { - return __builtin_popcountll(input_num); -} -#endif - - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_BITMANIPULATION_H -/* end file simdjson/ppc64/bitmanipulation.h */ -/* including simdjson/ppc64/bitmask.h: #include "simdjson/ppc64/bitmask.h" */ -/* begin file simdjson/ppc64/bitmask.h */ -#ifndef SIMDJSON_PPC64_BITMASK_H -#define SIMDJSON_PPC64_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is -// encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { - // You can use the version below, however gcc sometimes miscompiles - // vec_pmsum_be, it happens somewhere around between 8 and 9th version. - // The performance boost was not noticeable, falling back to a usual - // implementation. - // __vector unsigned long long all_ones = {~0ull, ~0ull}; - // __vector unsigned long long mask = {bitmask, 0}; - // // Clang and GCC return different values for pmsum for ull so cast it to one. - // // Generally it is not specified by ALTIVEC ISA what is returned by - // // vec_pmsum_be. - // #if defined(__LITTLE_ENDIAN__) - // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[0]); - // #else - // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[1]); - // #endif - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; - return bitmask; -} - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif -/* end file simdjson/ppc64/bitmask.h */ -/* including simdjson/ppc64/numberparsing_defs.h: #include "simdjson/ppc64/numberparsing_defs.h" */ -/* begin file simdjson/ppc64/numberparsing_defs.h */ -#ifndef SIMDJSON_PPC64_NUMBERPARSING_DEFS_H -#define SIMDJSON_PPC64_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#if defined(__linux__) -#include -#elif defined(__FreeBSD__) -#include -#endif - -namespace simdjson { -namespace ppc64 { -namespace numberparsing { - -// we don't have appropriate instructions, so let us use a scalar function -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - uint64_t val; - std::memcpy(&val, chars, sizeof(uint64_t)); -#ifdef __BIG_ENDIAN__ -#if defined(__linux__) - val = bswap_64(val); -#elif defined(__FreeBSD__) - val = bswap64(val); -#endif -#endif - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace ppc64 -} // namespace simdjson - -#ifndef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_IS_BIG_ENDIAN -#define SIMDJSON_SWAR_NUMBER_PARSING 0 -#else -#define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif -#endif - -#endif // SIMDJSON_PPC64_NUMBERPARSING_DEFS_H -/* end file simdjson/ppc64/numberparsing_defs.h */ -/* including simdjson/ppc64/simd.h: #include "simdjson/ppc64/simd.h" */ -/* begin file simdjson/ppc64/simd.h */ -#ifndef SIMDJSON_PPC64_SIMD_H -#define SIMDJSON_PPC64_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace ppc64 { -namespace { -namespace simd { - -using __m128i = __vector unsigned char; - -template struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{__m128i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i &() const { - return this->value; - } - simdjson_inline operator __m128i &() { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { - return vec_or(this->value, (__m128i)other); - } - simdjson_inline Child operator&(const Child other) const { - return vec_and(this->value, (__m128i)other); - } - simdjson_inline Child operator^(const Child other) const { - return vec_xor(this->value, (__m128i)other); - } - simdjson_inline Child bit_andnot(const Child other) const { - return vec_andc(this->value, (__m128i)other); - } - simdjson_inline Child &operator|=(const Child other) { - auto this_cast = static_cast(this); - *this_cast = *this_cast | other; - return *this_cast; - } - simdjson_inline Child &operator&=(const Child other) { - auto this_cast = static_cast(this); - *this_cast = *this_cast & other; - return *this_cast; - } - simdjson_inline Child &operator^=(const Child other) { - auto this_cast = static_cast(this); - *this_cast = *this_cast ^ other; - return *this_cast; - } -}; - -template > -struct base8 : base> { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { - return (__m128i)vec_cmpeq(lhs.value, (__m128i)rhs); - } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(simd8 prev_chunk) const { - __m128i chunk = this->value; -#ifdef __LITTLE_ENDIAN__ - chunk = (__m128i)vec_reve(this->value); - prev_chunk = (__m128i)vec_reve((__m128i)prev_chunk); -#endif - chunk = (__m128i)vec_sld((__m128i)prev_chunk, (__m128i)chunk, 16 - N); -#ifdef __LITTLE_ENDIAN__ - chunk = (__m128i)vec_reve((__m128i)chunk); -#endif - return chunk; - } -}; - -// SIMD byte mask type (returned by things like eq and gt) -template <> struct simd8 : base8 { - static simdjson_inline simd8 splat(bool _value) { - return (__m128i)vec_splats((unsigned char)(-(!!_value))); - } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) - : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) - : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { - __vector unsigned long long result; - const __m128i perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40, - 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00}; - - result = ((__vector unsigned long long)vec_vbpermq((__m128i)this->value, - (__m128i)perm_mask)); -#ifdef __LITTLE_ENDIAN__ - return static_cast(result[1]); -#else - return static_cast(result[0]); -#endif - } - simdjson_inline bool any() const { - return !vec_all_eq(this->value, (__m128i)vec_splats(0)); - } - simdjson_inline simd8 operator~() const { - return this->value ^ (__m128i)splat(true); - } -}; - -template struct base8_numeric : base8 { - static simdjson_inline simd8 splat(T value) { - (void)value; - return (__m128i)vec_splats(value); - } - static simdjson_inline simd8 zero() { return splat(0); } - static simdjson_inline simd8 load(const T values[16]) { - return (__m128i)(vec_vsx_ld(0, reinterpret_cast(values))); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, - T v5, T v6, T v7, T v8, T v9, - T v10, T v11, T v12, T v13, - T v14, T v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, - v14, v15); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) - : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { - vec_vsx_st(this->value, 0, reinterpret_cast<__m128i *>(dst)); - } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { - return (__m128i)((__m128i)this->value + (__m128i)other); - } - simdjson_inline simd8 operator-(const simd8 other) const { - return (__m128i)((__m128i)this->value - (__m128i)other); - } - simdjson_inline simd8 &operator+=(const simd8 other) { - *this = *this + other; - return *static_cast *>(this); - } - simdjson_inline simd8 &operator-=(const simd8 other) { - *this = *this - other; - return *static_cast *>(this); - } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior - // for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return (__m128i)vec_perm((__m128i)lookup_table, (__m128i)lookup_table, this->value); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted - // as a bitset). Passing a 0 value for mask would be equivalent to writing out - // every byte to output. Only the first 16 - count_ones(mask) bytes of the - // result are significant but 16 bytes get written. Design consideration: it - // seems like a function with the signature simd8 compress(uint32_t mask) - // would be sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L *output) const { - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - using internal::thintable_epi8; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. -#ifdef __LITTLE_ENDIAN__ - __m128i shufmask = (__m128i)(__vector unsigned long long){ - thintable_epi8[mask1], thintable_epi8[mask2]}; -#else - __m128i shufmask = (__m128i)(__vector unsigned long long){ - thintable_epi8[mask2], thintable_epi8[mask1]}; - shufmask = (__m128i)vec_reve((__m128i)shufmask); -#endif - // we increment by 0x08 the second half of the mask - shufmask = ((__m128i)shufmask) + - ((__m128i)(__vector int){0, 0, 0x08080808, 0x08080808}); - - // this is the version "nearly pruned" - __m128i pruned = vec_perm(this->value, this->value, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - __m128i compactmask = - vec_vsx_ld(0, reinterpret_cast(pshufb_combine_table + pop1 * 8)); - __m128i answer = vec_perm(pruned, (__m128i)vec_splats(0), compactmask); - vec_vsx_st(answer, 0, reinterpret_cast<__m128i *>(output)); - } - - template - simdjson_inline simd8 - lookup_16(L replace0, L replace1, L replace2, L replace3, L replace4, - L replace5, L replace6, L replace7, L replace8, L replace9, - L replace10, L replace11, L replace12, L replace13, L replace14, - L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, replace4, replace5, replace6, - replace7, replace8, replace9, replace10, replace11, replace12, - replace13, replace14, replace15)); - } -}; - -// Signed bytes -template <> struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) - : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t *values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3, - int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, - int8_t v12, int8_t v13, int8_t v14, int8_t v15) - : simd8((__m128i)(__vector signed char){v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, - v15}) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 - repeat_16(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, - int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, - int8_t v12, int8_t v13, int8_t v14, int8_t v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15); - } - - // Order-sensitive comparisons - simdjson_inline simd8 - max_val(const simd8 other) const { - return (__m128i)vec_max((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - simdjson_inline simd8 - min_val(const simd8 other) const { - return (__m128i)vec_min((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - simdjson_inline simd8 - operator>(const simd8 other) const { - return (__m128i)vec_cmpgt((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - simdjson_inline simd8 - operator<(const simd8 other) const { - return (__m128i)vec_cmplt((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } -}; - -// Unsigned bytes -template <> struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) - : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t *values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline - simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, - uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, - uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) - : simd8((__m128i){v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15}) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 - repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, - uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, - uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, - uint8_t v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15); - } - - // Saturated math - simdjson_inline simd8 - saturating_add(const simd8 other) const { - return (__m128i)vec_adds(this->value, (__m128i)other); - } - simdjson_inline simd8 - saturating_sub(const simd8 other) const { - return (__m128i)vec_subs(this->value, (__m128i)other); - } - - // Order-specific operations - simdjson_inline simd8 - max_val(const simd8 other) const { - return (__m128i)vec_max(this->value, (__m128i)other); - } - simdjson_inline simd8 - min_val(const simd8 other) const { - return (__m128i)vec_min(this->value, (__m128i)other); - } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 - gt_bits(const simd8 other) const { - return this->saturating_sub(other); - } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 - lt_bits(const simd8 other) const { - return other.saturating_sub(*this); - } - simdjson_inline simd8 - operator<=(const simd8 other) const { - return other.max_val(*this) == other; - } - simdjson_inline simd8 - operator>=(const simd8 other) const { - return other.min_val(*this) == other; - } - simdjson_inline simd8 - operator>(const simd8 other) const { - return this->gt_bits(other).any_bits_set(); - } - simdjson_inline simd8 - operator<(const simd8 other) const { - return this->gt_bits(other).any_bits_set(); - } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { - return (__m128i)vec_cmpeq(this->value, (__m128i)vec_splats(uint8_t(0))); - } - simdjson_inline simd8 bits_not_set(simd8 bits) const { - return (*this & bits).bits_not_set(); - } - simdjson_inline simd8 any_bits_set() const { - return ~this->bits_not_set(); - } - simdjson_inline simd8 any_bits_set(simd8 bits) const { - return ~this->bits_not_set(bits); - } - simdjson_inline bool bits_not_set_anywhere() const { - return vec_all_eq(this->value, (__m128i)vec_splats(0)); - } - simdjson_inline bool any_bits_set_anywhere() const { - return !bits_not_set_anywhere(); - } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { - return vec_all_eq(vec_and(this->value, (__m128i)bits), - (__m128i)vec_splats(0)); - } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { - return !bits_not_set_anywhere(bits); - } - template simdjson_inline simd8 shr() const { - return simd8( - (__m128i)vec_sr(this->value, (__m128i)vec_splat_u8(N))); - } - template simdjson_inline simd8 shl() const { - return simd8( - (__m128i)vec_sl(this->value, (__m128i)vec_splat_u8(N))); - } -}; - -template struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, - "PPC64 kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64 &o) = delete; // no copy allowed - simd8x64 & - operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, - const simd8 chunk2, const simd8 chunk3) - : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_inline simd8x64(const T ptr[64]) - : chunks{simd8::load(ptr), simd8::load(ptr + 16), - simd8::load(ptr + 32), simd8::load(ptr + 48)} {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr + sizeof(simd8) * 0); - this->chunks[1].store(ptr + sizeof(simd8) * 1); - this->chunks[2].store(ptr + sizeof(simd8) * 2); - this->chunks[3].store(ptr + sizeof(simd8) * 3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | - (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t compress(uint64_t mask, T *output) const { - this->chunks[0].compress(uint16_t(mask), output); - this->chunks[1].compress(uint16_t(mask >> 16), - output + 16 - count_ones(mask & 0xFFFF)); - this->chunks[2].compress(uint16_t(mask >> 32), - output + 32 - count_ones(mask & 0xFFFFFFFF)); - this->chunks[3].compress(uint16_t(mask >> 48), - output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); - return 64 - count_ones(mask); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r1 = this->chunks[1].to_bitmask(); - uint64_t r2 = this->chunks[2].to_bitmask(); - uint64_t r3 = this->chunks[3].to_bitmask(); - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] == mask, this->chunks[1] == mask, - this->chunks[2] == mask, this->chunks[3] == mask) - .to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64(this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3]) - .to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] <= mask, this->chunks[1] <= mask, - this->chunks[2] <= mask, this->chunks[3] <= mask) - .to_bitmask(); - } -}; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_SIMD_INPUT_H -/* end file simdjson/ppc64/simd.h */ -/* including simdjson/ppc64/stringparsing_defs.h: #include "simdjson/ppc64/stringparsing_defs.h" */ -/* begin file simdjson/ppc64/stringparsing_defs.h */ -#ifndef SIMDJSON_PPC64_STRINGPARSING_DEFS_H -#define SIMDJSON_PPC64_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/simd.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote - copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { - return ((bs_bits - 1) & quote_bits) != 0; - } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { - return trailing_zeroes(quote_bits); - } - simdjson_inline int backslash_index() { - return trailing_zeroes(bs_bits); - } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote -backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), - "backslash and quote finder must process fewer than " - "SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + sizeof(v0)); - v0.store(dst); - v1.store(dst + sizeof(v0)); - - // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on - // PPC; therefore, we smash them together into a 64-byte mask and get the - // bitmask from there. - uint64_t bs_and_quote = - simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; -} - - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_STRINGPARSING_DEFS_H -/* end file simdjson/ppc64/stringparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 -/* end file simdjson/ppc64/begin.h */ -/* including generic/amalgamated.h for ppc64: #include */ -/* begin file generic/amalgamated.h for ppc64 */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) -#error generic/dependencies.h must be included before generic/amalgamated.h! -#endif - -/* including generic/base.h for ppc64: #include */ -/* begin file generic/base.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { - -struct json_character_block; - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_BASE_H -/* end file generic/base.h for ppc64 */ -/* including generic/dom_parser_implementation.h for ppc64: #include */ -/* begin file generic/dom_parser_implementation.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// Interface a dom parser implementation must fulfill -namespace simdjson { -namespace ppc64 { -namespace { - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); -simdjson_inline bool is_ascii(const simd8x64& input); - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file generic/dom_parser_implementation.h for ppc64 */ -/* including generic/json_character_block.h for ppc64: #include */ -/* begin file generic/json_character_block.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { - -struct json_character_block { - static simdjson_inline json_character_block classify(const simd::simd8x64& in); - - simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_inline uint64_t op() const noexcept { return _op; } - simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } - - uint64_t _whitespace; - uint64_t _op; -}; - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H -/* end file generic/json_character_block.h for ppc64 */ -/* end file generic/amalgamated.h for ppc64 */ -/* including generic/stage1/amalgamated.h for ppc64: #include */ -/* begin file generic/stage1/amalgamated.h for ppc64 */ -// Stuff other things depend on -/* including generic/stage1/base.h for ppc64: #include */ -/* begin file generic/stage1/base.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage1 { - -class bit_indexer; -template -struct buf_block_reader; -struct json_block; -class json_minifier; -class json_scanner; -struct json_string_block; -class json_string_scanner; -class json_structural_indexer; - -} // namespace stage1 - -namespace utf8_validation { -struct utf8_checker; -} // namespace utf8_validation - -using utf8_validation::utf8_checker; - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H -/* end file generic/stage1/base.h for ppc64 */ -/* including generic/stage1/buf_block_reader.h for ppc64: #include */ -/* begin file generic/stage1/buf_block_reader.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage1 { - -// Walks through a buffer in block-sized increments, loading the last part with spaces -template -struct buf_block_reader { -public: - simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t *full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t *dst) const; - /** - * Like get_remainder, for input followed by SIMDJSON_PADDING readable bytes. - * - * get_remainder writes the block with a memset and a memcpy of the remaining - * length, and the vector loads that read it back cannot be forwarded from - * those narrower stores: they wait until the stores reach the cache, which - * costs tens of cycles on every document. Here the block is put together in - * vector registers, from loads that may read into the padding, and written - * with full-width stores that the loads can forward from. - */ - simdjson_inline size_t get_remainder_padded(uint8_t *dst) const; - simdjson_inline void advance(); -private: - const uint8_t *buf; - const size_t len; - const size_t lenminusstep; - size_t idx; -}; - -template -simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} - -template -simdjson_inline size_t buf_block_reader::block_index() { return idx; } - -template -simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; -} - -template -simdjson_inline const uint8_t *buf_block_reader::full_block() const { - return &buf[idx]; -} - -template -simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { - if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; -} - -// keep_mask[128 - n + i] is 0xFF for i < n and 0 otherwise. -alignas(64) static const uint8_t keep_mask[256] = { - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF -}; - -template -simdjson_inline size_t buf_block_reader::get_remainder_padded(uint8_t *dst) const { - static_assert(STEP_SIZE <= 128, "keep_mask covers blocks of up to 128 bytes"); - static_assert(sizeof(simd::simd8) <= SIMDJSON_PADDING, "a vector load at the last byte must stay in the padding"); - const size_t remaining = len - idx; - if (remaining == 0) { return 0; } - const simd::simd8 spaces = simd::simd8::splat(0x20); - for (size_t off = 0; off < STEP_SIZE; off += sizeof(simd::simd8)) { - simd::simd8 v = spaces; - if (off < remaining) { - // Starts before len, so it ends within the padding. - const simd::simd8 in = simd::simd8::load(buf + idx + off); - const simd::simd8 keep = simd::simd8::load(keep_mask + 128 - remaining + off); - v = spaces ^ ((in ^ spaces) & keep); - } - v.store(dst + off); - } - return remaining; -} - -template -simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for ppc64 */ -/* including generic/stage1/json_escape_scanner.h for ppc64: #include */ -/* begin file generic/stage1/json_escape_scanner.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage1 { - -/** - * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). - */ -struct json_escape_scanner { - /** The actual escape characters (the backslashes themselves). */ - uint64_t next_is_escaped = 0ULL; - - struct escaped_and_escape { - /** - * Mask of escaped characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 0100100010100101000 - * n \ \ n \ \ - * ``` - */ - uint64_t escaped; - /** - * Mask of escape characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 1001000101001010001 - * \ \ \ \ \ \ \ - * ``` - */ - uint64_t escape; - }; - - /** - * Get a mask of both escape and escaped characters (the characters following a backslash). - * - * @param potential_escape A mask of the character that can escape others (but could be - * escaped itself). e.g. block.eq('\\') - */ - simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { - -#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT - if (!backslash) { return {next_escaped_without_backslashes(), 0}; } -#endif - - // | | Mask (shows characters instead of 1's) | Depth | Instructions | - // |--------------------------------|----------------------------------------|-------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | - // | | ` even odd even odd odd` | | | - // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) - // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) - // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) - // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () - // (*) this is not needed until the next iteration - uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); - uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); - uint64_t escape = escape_and_terminal_code & backslash; - this->next_is_escaped = escape >> 63; - return {escaped, escape}; - } - -private: - static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; - - simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { - uint64_t escaped = this->next_is_escaped; - this->next_is_escaped = 0; - return escaped; - } - - /** - * Returns a mask of the next escape characters (masking out escaped backslashes), along with - * any non-backslash escape codes. - * - * \n \\n \\\n \\\\n returns: - * \n \ \ \n \ \ - * 11 100 1011 10100 - * - * You are expected to mask out the first bit yourself if the previous block had a trailing - * escape. - * - * & the result with potential_escape to get just the escape characters. - * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. - */ - static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { - // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: - // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be - // inverted (\\\ would be 010 instead of 101). - // - // ``` - // string: | ____\\\\_\\\\_____ | - // maybe_escaped | ODD | \ \ \ \ | - // even-aligned ^^^ ^^^^ odd-aligned - // ``` - // - // Taking that into account, our basic strategy is: - // - // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for - // odd-aligned runs. - // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the - // odd bits in odd-aligned runs. - // 3. & with backslash to clean up any stray bits. - // runs are set to 0, and then XORing with "odd": - // - // | | Mask (shows characters instead of 1's) | Instructions | - // |--------------------------------|----------------------------------------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | - // | | ` even odd even odd odd` | - // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) - // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) - // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) - // - - // Escaped characters are characters following an escape. - uint64_t maybe_escaped = potential_escape << 1; - - // To distinguish odd from even escape sequences, therefore, we turn on any *starting* - // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) - // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. - // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. - // - All other odd bytes are 1, and even bytes are 0. - uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; - uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; - - // Now we flip all odd bytes back with xor. This: - // - Makes odd runs of backslashes go from 0000 to 1010 - // - Makes even runs of backslashes go from 1111 to 1010 - // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) - // - Resets all other bytes to 0 - return even_series_codes_and_odd_bits ^ ODD_BITS; - } -}; - -} // namespace stage1 -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_escape_scanner.h for ppc64 */ -/* including generic/stage1/json_string_scanner.h for ppc64: #include */ -/* begin file generic/stage1/json_string_scanner.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage1 { - -struct json_string_block { - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : - _escaped(escaped), _quote(quote), _in_string(in_string) {} - - // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } - // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } - // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } - // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } - - // escaped characters (backslashed--does not include the hex characters after \u) - uint64_t _escaped; - // real quotes (non-escaped ones) - uint64_t _quote; - // string characters (includes start quote but not end quote) - uint64_t _in_string; -}; - -// Scans blocks for string characters, storing the state necessary to do so -class json_string_scanner { -public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); - -private: - // Scans for escape characters - json_escape_scanner escape_scanner{}; - // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). - uint64_t prev_in_string = 0ULL; -}; - -// -// Return a mask of all string characters plus end quotes. -// -// prev_escaped is overflow saying whether the next character is escaped. -// prev_in_string is overflow saying whether we're still in a string. -// -// Backslash sequences outside of quotes will be detected in stage 2. -// -simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = escape_scanner.next(backslash).escaped; - const uint64_t quote = in.eq('"') & ~escaped; - - // - // prefix_xor flips on bits inside the string (and flips off the end quote). - // - // Then we xor with prev_in_string: if we were in a string already, its effect is flipped - // (characters inside strings are outside, and characters outside strings are inside). - // - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - - // - // Check if we're still in a string at the end of the box so the next block will know - // - prev_in_string = uint64_t(static_cast(in_string) >> 63); - - // Use ^ to turn the beginning quote off, and the end quote on. - - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_string_block(escaped, quote, in_string); -} - -simdjson_really_inline error_code json_string_scanner::finish() { - if (prev_in_string) { - return UNCLOSED_STRING; - } - return SUCCESS; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_string_scanner.h for ppc64 */ -/* including generic/stage1/utf8_lookup4_algorithm.h for ppc64: #include */ -/* begin file generic/stage1/utf8_lookup4_algorithm.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { -namespace utf8_validation { - -using namespace simd; - - simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { -// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) -// Bit 1 = Too Long (ASCII followed by continuation) -// Bit 2 = Overlong 3-byte -// Bit 4 = Surrogate -// Bit 5 = Overlong 2-byte -// Bit 7 = Two Continuations - constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ - // 11______ 11______ - constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ - constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ - constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ - constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ - constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ - constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ - // 11110100 101_____ - // 11110101 1001____ - // 11110101 101_____ - // 1111011_ 1001____ - // 1111011_ 101_____ - // 11111___ 1001____ - // 11111___ 101_____ - constexpr const uint8_t TOO_LARGE_1000 = 1<<6; - // 11110101 1000____ - // 1111011_ 1000____ - // 11111___ 1000____ - constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ - - const simd8 byte_1_high = prev1.shr<4>().lookup_16( - // 0_______ ________ - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - // 10______ ________ - TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, - // 1100____ ________ - TOO_SHORT | OVERLONG_2, - // 1101____ ________ - TOO_SHORT, - // 1110____ ________ - TOO_SHORT | OVERLONG_3 | SURROGATE, - // 1111____ ________ - TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 - ); - constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( - // ____0000 ________ - CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, - // ____0001 ________ - CARRY | OVERLONG_2, - // ____001_ ________ - CARRY, - CARRY, - - // ____0100 ________ - CARRY | TOO_LARGE, - // ____0101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____011_ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - - // ____1___ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____1101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000 - ); - const simd8 byte_2_high = input.shr<4>().lookup_16( - // ________ 0_______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - - // ________ 1000____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, - // ________ 1001____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, - // ________ 101_____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - - // ________ 11______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT - ); - return (byte_1_high & byte_1_low & byte_2_high); - } - simdjson_inline simd8 check_multibyte_lengths(const simd8 input, - const simd8 prev_input, const simd8 sc) { - simd8 prev2 = input.prev<2>(prev_input); - simd8 prev3 = input.prev<3>(prev_input); - simd8 must23 = must_be_2_3_continuation(prev2, prev3); - simd8 must23_80 = must23 & uint8_t(0x80); - return must23_80 ^ sc; - } - - // - // Return nonzero if there are incomplete multibyte characters at the end of the block: - // e.g. if there is a 4-byte character, but it's 3 bytes from the end. - // - simdjson_inline simd8 is_incomplete(const simd8 input) { - // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): - // ... 1111____ 111_____ 11______ -#if SIMDJSON_IMPLEMENTATION_ICELAKE || (SIMDJSON_IMPLEMENTATION_RVV_VLS && __riscv_v_fixed_vlen >= 512) - static const uint8_t max_array[64] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#else - static const uint8_t max_array[32] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#endif - const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); - return input.gt_bits(max_value); - } - - struct utf8_checker { - // If this is nonzero, there has been a UTF-8 error. - simd8 error; - // The last input we received - simd8 prev_input_block; - // Whether the last input we received was incomplete (used for ASCII fast path) - simd8 prev_incomplete; - - // - // Check whether the current bytes are valid UTF-8. - // - simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { - // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes - // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) - simd8 prev1 = input.prev<1>(prev_input); - simd8 sc = check_special_cases(input, prev1); - this->error |= check_multibyte_lengths(input, prev_input, sc); - } - - // The only problem that can happen at EOF is that a multibyte character is too short - // or a byte value too large in the last bytes: check_special_cases only checks for bytes - // too large in the first of two bytes. - simdjson_inline void check_eof() { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; - } - - simdjson_inline void check_next_input(const simd8x64& input) { - if(simdjson_likely(is_ascii(input))) { - this->error |= this->prev_incomplete; - } else { - // you might think that a for-loop would work, but under Visual Studio, it is not good enough. - static_assert((simd8x64::NUM_CHUNKS == 1) - ||(simd8x64::NUM_CHUNKS == 2) - || (simd8x64::NUM_CHUNKS == 4), - "We support one, two or four chunks per 64-byte block."); - SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 1) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 2) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 4) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - this->check_utf8_bytes(input.get<2>(), input.get<1>()); - this->check_utf8_bytes(input.get<3>(), input.get<2>()); - } - this->prev_incomplete = is_incomplete(input.get::NUM_CHUNKS-1>()); - this->prev_input_block = input.get::NUM_CHUNKS-1>(); - } - } - // do not forget to call check_eof! - simdjson_warn_unused simdjson_inline error_code errors() { - return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; - } - - }; // struct utf8_checker -} // namespace utf8_validation - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H -/* end file generic/stage1/utf8_lookup4_algorithm.h for ppc64 */ -/* including generic/stage1/json_scanner.h for ppc64: #include */ -/* begin file generic/stage1/json_scanner.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage1 { - -/** - * A block of scanned json, with information on operators and scalars. - * - * We seek to identify pseudo-structural characters. Anything that is inside - * a string must be omitted (hence & ~_string.string_tail()). - * Otherwise, pseudo-structural characters come in two forms. - * 1. We have the structural characters ([,],{,},:, comma). The - * term 'structural character' is from the JSON RFC. - * 2. We have the 'scalar pseudo-structural characters'. - * Scalars are quotes, and any character except structural characters and white space. - * - * To identify the scalar pseudo-structural characters, we must look at what comes - * before them: it must be a space, a quote or a structural characters. - * Starting with simdjson v0.3, we identify them by - * negation: we identify everything that is followed by a non-quote scalar, - * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. - */ -struct json_block { -public: - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - - /** - * The start of structurals. - * In simdjson prior to v0.3, these were called the pseudo-structural characters. - **/ - simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } - /** All JSON whitespace (i.e. not in a string) */ - simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } - - // Helpers - - /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } - /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } - - // string and escape characters - json_string_block _string; - // whitespace, structural characters ('operators'), scalars - json_character_block _characters; - // whether the previous character was a scalar - uint64_t _follows_potential_nonquote_scalar; -private: - // Potential structurals (i.e. disregarding strings) - - /** - * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". - * They may reside inside a string. - **/ - simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } - /** - * The start of non-operator runs, like 123, true and "abc". - * It main reside inside a string. - **/ - simdjson_inline uint64_t potential_scalar_start() const noexcept { - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space - // then we know that it is irrelevant structurally. - return _characters.scalar() & ~follows_potential_scalar(); - } - /** - * Whether the given character is immediately after a non-operator like 123, true. - * The characters following a quote are not included. - */ - simdjson_inline uint64_t follows_potential_scalar() const noexcept { - // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character - // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a - // white space. - // It is understood that within quoted region, anything at all could be marked (irrelevant). - return _follows_potential_nonquote_scalar; - } -}; - -/** - * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. - * - * The scanner starts by calculating two distinct things: - * - string characters (taking \" into account) - * - structural characters or 'operators' ([]{},:, comma) - * and scalars (runs of non-operators like 123, true and "abc") - * - * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: - * in particular, the operator/scalar bit will find plenty of things that are actually part of - * strings. When we're done, json_block will fuse the two together by masking out tokens that are - * part of a string. - */ -class json_scanner { -public: - json_scanner() = default; - simdjson_inline json_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_warn_unused simdjson_inline error_code finish(); - -private: - // Whether the last character of the previous iteration is part of a scalar token - // (anything except whitespace or a structural character/'operator'). - uint64_t prev_scalar = 0ULL; - json_string_scanner string_scanner{}; -}; - - -// -// Check if the current character immediately follows a matching character. -// -// For example, this checks for quotes with backslashes in front of them: -// -// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); -// -simdjson_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; -} - -simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { - json_string_block strings = string_scanner.next(in); - // identifies the white-space and the structural characters - json_character_block characters = json_character_block::classify(in); - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). - // - // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) - // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential - // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we - // may need to add an extra check when parsing strings. - // - // Performance: there are many ways to skin this cat. - const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); - uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_block( - strings,// strings is a function-local object so either it moves or the copy is elided. - characters, - follows_nonquote_scalar - ); -} - -simdjson_warn_unused simdjson_inline error_code json_scanner::finish() { - return string_scanner.finish(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H -/* end file generic/stage1/json_scanner.h for ppc64 */ - -// All other declarations -/* including generic/stage1/find_next_document_index.h for ppc64: #include */ -/* begin file generic/stage1/find_next_document_index.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage1 { - -/** - * This algorithm is used to quickly identify the last structural position that - * makes up a complete document. - * - * It does this by going backwards and finding the last *document boundary* (a - * place where one value follows another without a comma between them). If the - * last document (the characters after the boundary) has an equal number of - * start and end brackets, it is considered complete. - * - * Simply put, we iterate over the structural characters, starting from - * the end. We consider that we found the end of a JSON document when the - * first element of the pair is NOT one of these characters: '{' '[' ':' ',' - * and when the second element is NOT one of these characters: '}' ']' ':' ','. - * - * This simple comparison works most of the time, but it does not cover cases - * where the batch's structural indexes contain a perfect amount of documents. - * In such a case, we do not have access to the structural index which follows - * the last document, therefore, we do not have access to the second element in - * the pair, and that means we cannot identify the last document. To fix this - * issue, we keep a count of the open and closed curly/square braces we found - * while searching for the pair. When we find a pair AND the count of open and - * closed curly/square braces is the same, we know that we just passed a - * complete document, therefore the last json buffer location is the end of the - * batch. - */ -simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { - const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - const uint8_t e = parser.buf[len - 1]; - return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && - e != ' ' && e != '\t' && e != '\n' && e != '\r'; -} - -/** - * The structural index past the last one of a partial batch is the end of this - * window, not a token of it: the byte there is the first byte of the next - * window. A finished value must not be rejected because that byte is ',', ':', - * ']' or '}'. A last document that itself ends with '{', '[', ',' or ':' is - * invalid JSON, yet its brackets may balance: walking it, the On-Demand API - * could read the next batch as a value. We leave such a document to the next - * batch, so that the batch ends between two documents. - */ -simdjson_inline bool ends_with_unfinished_document(dom_parser_implementation &parser) { - const uint8_t last = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - return last == '{' || last == '[' || last == ',' || last == ':'; -} - -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { - // Variant: do not count separately, just figure out depth - if(parser.n_structural_indexes == 0) { return 0; } - auto arr_cnt = 0; - auto obj_cnt = 0; - for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { - auto idxb = parser.structural_indexes[i]; - switch (parser.buf[idxb]) { - case ':': - case ',': - continue; - case '}': - obj_cnt--; - continue; - case ']': - arr_cnt--; - continue; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - auto idxa = parser.structural_indexes[i - 1]; - switch (parser.buf[idxa]) { - case '{': - case '[': - case ':': - case ',': - continue; - } - // Last document is complete, so the next document will appear after! - if (!arr_cnt && !obj_cnt) { - return defer_last ? i : parser.n_structural_indexes; - } - // Last document is incomplete; mark the document at i + 1 as the next one - return i; - } - // If we made it to the end, we want to finish counting to see if we have a full document. - switch (parser.buf[parser.structural_indexes[0]]) { - case '}': - obj_cnt--; - break; - case ']': - arr_cnt--; - break; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - if (!arr_cnt && !obj_cnt) { - // We have a complete document. - return defer_last ? 0 : parser.n_structural_indexes; - } - return 0; -} - -/** - * Sentinel value returned to indicate a document started but didn't fit - * (CAPACITY error), as opposed to 0 which means no document content found - * (EMPTY). - */ -constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; - -/** - * For RFC 7464 JSON text sequences, filter RS from structural indexes and - * find batch boundaries. - * - * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. - * RS bytes appear in structural_indexes as they are classified as scalars. - * This function: - * 1. Scans structural_indexes to find and count RS positions - * 2. Filters RS out of structural_indexes in-place - * 3. Determines batch boundaries based on RS positions - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @param scan_len Offset past which no value start may be derived. Equals len - * unless stage 1 dropped a trailing unclosed string, whose bytes it - * never validated. - * @return The number of structural indexes to keep (after RS filtering), - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t find_next_document_index_json_sequence( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start, - size_t scan_len) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Phase 1: Scan structural_indexes to find RS positions and handle them. - // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS - // is already in structural_indexes (it's an operator). For scalars like numbers, - // the digit following RS is NOT in structural_indexes because the scanner sees - // RS as a scalar, making the digit a scalar continuation, not a start. - // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the - // actual value start position. - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_rs_pos = 0; - uint32_t rs_count = 0; - - for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { - const uint32_t pos = parser.structural_indexes[read_idx]; - if (parser.buf[pos] == 0x1E) { - // This is an RS character - find the actual JSON value start. - last_rs_pos = pos; - rs_count++; - // Skip past this RS and any whitespace *and any additional RSes* - // to locate the real value. Consecutive RSes are degenerate - // "empty records" per RFC 7464; we collapse them here. They do - // not always appear as separate entries in structural_indexes - // because the scanner groups runs of adjacent non-whitespace - // scalar bytes (including RS) into a single scalar start. - uint32_t value_start = pos + 1; - while (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { - value_start++; - } else if (c == 0x1E) { - // Collapsed empty record. Still count it so rs_count reflects - // the true number of record markers and last_rs_pos tracks - // the final one. - last_rs_pos = value_start; - rs_count++; - value_start++; - } else { - break; - } - } - // If the scanner emitted additional structurals inside the - // whitespace+RS run we just walked over (i.e., isolated RSes - // separated by whitespace), skip past them so we do not - // double-count or double-emit. - while (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] < value_start) { - read_idx++; - } - // Check if the value start is an operator (always present in - // scanner structural_indexes) or a scalar-like start (which may - // be missing from structural_indexes and must be added here). - // Note: '"' is NOT always in structural_indexes. The scanner - // classifies '"' as a scalar character and emits it as a - // structural only when it is a *scalar start* (preceded by - // whitespace or an operator). When '"' immediately follows an - // RS (which the scanner also classifies as scalar), it is - // treated as a scalar continuation and not emitted - so we - // must add it here just like any other scalar value. - if (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - const bool is_operator = - (c == '{' || c == '}' || c == '[' || c == ']' || - c == ':' || c == ','); - // If the next scanner structural is exactly at value_start, - // the scanner already emitted it (it followed whitespace) and - // we must not add a duplicate - a subsequent iteration will - // copy it into write_idx. - const bool already_emitted = - (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] == value_start); - if (!is_operator && !already_emitted) { - // Scalar value (number/true/false/null/string) - add its - // position since scanner missed it. - parser.structural_indexes[write_idx++] = value_start; - } - } - } else { - // Not RS, copy to output - parser.structural_indexes[write_idx++] = pos; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { - // Only RS markers here: the last one opens a record continuing past the - // window, so that is where the next batch resumes. - if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } - return 0; - } - if (rs_count == 0) { - // No RS found; for final batch, try generic boundary detection - return is_final ? find_next_document_index(parser) : 0; - } - - // Phase 2: Determine batch boundaries based on RS positions - - if (is_final) { - // Final batch: all documents are complete (last one ends at EOF). - // In json_sequence mode, RS markers define document boundaries, so all - // remaining structurals form complete documents. Return them all directly. - // (Calling find_next_document_index() would fail for scalar documents.) - return parser.n_structural_indexes; - } - - // Partial batch: need to find complete documents only. - // A document starting at an RS is complete if there is another RS after it. - next_batch_start = last_rs_pos; - - if (rs_count < 2) { - // Only one RS, so we have at most one document that may be incomplete. - // We cannot confirm it is complete without another RS. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least 2 RS markers. The last complete document ends before last_rs_pos. - - // Find the structural index cutoff: keep only structurals < last_rs_pos. - // Since we already filtered RS, all remaining structurals are valid. - // We iterate backward to find the last structural before last_rs_pos. - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_rs_pos) { - keep_count = i; - break; - } - } - - // No structurals before the last RS - no complete documents - if (keep_count == 0) { return 0; } - - // All documents before the last RS are complete by definition (the next RS - // confirms their end). No need to call find_next_document_index() which - // would fail for scalar documents like `1` or `"hello"`. - return keep_count; -} - -/** - * Filter comma-delimited documents by removing root-level commas from - * structural indexes. - * - * For comma-delimited format like `{...},{...},{...}`, we need to remove - * the commas that separate documents (depth 0) while preserving commas - * inside arrays and objects (depth > 0). - * - * After filtering, the structural indexes look like whitespace-delimited - * documents, so find_next_document_index() works unchanged. - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @return The number of structural indexes to keep, - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t filter_comma_delimited( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Track depth to identify root-level commas (depth 0) - int depth = 0; - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_root_comma_pos = 0; - uint32_t root_comma_count = 0; - - for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { - uint32_t idx = parser.structural_indexes[i]; - uint8_t c = parser.buf[idx]; - - switch (c) { - case '{': case '[': - depth++; - parser.structural_indexes[write_idx++] = idx; - break; - case '}': case ']': - depth--; - parser.structural_indexes[write_idx++] = idx; - break; - case ',': - if (depth == 0) { - // Root-level comma = document boundary, skip it - last_root_comma_pos = idx; - root_comma_count++; - continue; - } - parser.structural_indexes[write_idx++] = idx; - break; - default: - // Colons, scalars, etc. - parser.structural_indexes[write_idx++] = idx; - break; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { return 0; } - - if (is_final) { - // Final batch: use standard boundary detection on filtered indexes - return find_next_document_index(parser); - } - - // Partial batch: need to find complete documents only. - // A document ending with a root comma is complete. - if (root_comma_count == 0) { - // No root commas found; we cannot confirm any document is complete. - // The whole batch might be one incomplete document. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least one root comma. Documents before the last comma are complete. - next_batch_start = last_root_comma_pos + 1; - - // Find the structural index cutoff: keep only structurals < last_root_comma_pos - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_root_comma_pos) { - keep_count = i; - break; - } - } - - if (keep_count == 0) { return 0; } - - // Use standard boundary detection on the complete portion - parser.n_structural_indexes = keep_count; - return find_next_document_index(parser); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H -/* end file generic/stage1/find_next_document_index.h for ppc64 */ -/* including generic/stage1/json_minifier.h for ppc64: #include */ -/* begin file generic/stage1/json_minifier.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage1 { - -class json_minifier { -public: - template - static error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept; - -private: - simdjson_inline json_minifier(uint8_t *_dst) - : dst{_dst} - {} - template - simdjson_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_warn_unused simdjson_inline error_code finish(uint8_t *dst_start, size_t &dst_len); - json_scanner scanner{}; - uint8_t *dst; -}; - -simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { - uint64_t mask = block.whitespace(); - dst += in.compress(mask, dst); -} - -simdjson_warn_unused simdjson_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { - error_code error = scanner.finish(); - if (error) { dst_len = 0; return error; } - dst_len = dst - dst_start; - return SUCCESS; -} - -template<> -simdjson_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - simd::simd8x64 in_2(block_buf+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1); - this->next(in_2, block_2); - reader.advance(); -} - -template<> -simdjson_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - json_block block_1 = scanner.next(in_1); - this->next(block_buf, block_1); - reader.advance(); -} - -template -error_code json_minifier::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept { - buf_block_reader reader(buf, len); - json_minifier minifier(dst); - - // Index the first n-1 blocks - while (reader.has_full_block()) { - minifier.step(reader.full_block(), reader); - } - - // Index the last (remainder) block, padded with spaces - uint8_t block[STEP_SIZE]; - size_t remaining_bytes = reader.get_remainder(block); - if (remaining_bytes > 0) { - // We do not want to write directly to the output stream. Rather, we write - // to a local buffer (for safety). - uint8_t out_block[STEP_SIZE]; - uint8_t * const guarded_dst{minifier.dst}; - minifier.dst = out_block; - minifier.step(block, reader); - size_t to_write = minifier.dst - out_block; - // In some cases, we could be enticed to consider the padded spaces - // as part of the string. This is fine as long as we do not write more - // than we consumed. - if(to_write > remaining_bytes) { to_write = remaining_bytes; } - memcpy(guarded_dst, out_block, to_write); - minifier.dst = guarded_dst + to_write; - } - return minifier.finish(dst, dst_len); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H -/* end file generic/stage1/json_minifier.h for ppc64 */ -/* including generic/stage1/json_structural_indexer.h for ppc64: #include */ -/* begin file generic/stage1/json_structural_indexer.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage1 { - -class bit_indexer { -public: - uint32_t *tail; - - simdjson_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} - -#if SIMDJSON_PREFER_REVERSE_BITS - /** - * ARM lacks a fast trailing zero instruction, but it has a fast - * bit reversal instruction and a fast leading zero instruction. - * Thus it may be profitable to reverse the bits (once) and then - * to rely on a sequence of instructions that call the leading - * zero instruction. - * - * Performance notes: - * The chosen routine is not optimal in terms of data dependency - * since zero_leading_bit might require two instructions. However, - * it tends to minimize the total number of instructions which is - * beneficial. - */ - simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { - int lz = leading_zeroes(rev_bits); - this->tail[i] = static_cast(idx) + lz; - rev_bits = zero_leading_bit(rev_bits, lz); - } -#else - /** - * Under recent x64 systems, we often have both a fast trailing zero - * instruction and a fast 'clear-lower-bit' instruction so the following - * algorithm can be competitive. - */ - - simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { - this->tail[i] = idx + trailing_zeroes(bits); - bits = clear_lowest_bit(bits); - } -#endif // SIMDJSON_PREFER_REVERSE_BITS - - template - simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { - write_index(idx, bits, START); - SIMDJSON_IF_CONSTEXPR (N > 1) { - write_indexes<(N-1>0?START+1:START), (N-1>=0?N-1:1)>(idx, bits); - } - return START+N; - } - - template - simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { - write_indexes(idx, bits); - SIMDJSON_IF_CONSTEXPR ((START+STEP) < END) { - if (simdjson_unlikely((START+STEP) < cnt)) { - write_indexes_stepped<(START+STEP(idx, bits, cnt); - } - } - return ((END-START) % STEP) == 0 ? END : (END-START) - ((END-START) % STEP) + STEP; - } - - // flatten out values in 'bits' assuming that they are are to have values of idx - // plus their position in the bitvector, and store these indexes at - // base_ptr[base] incrementing base as we go - // will potentially store extra values beyond end of valid bits, so base_ptr - // needs to be large enough to handle this - // - // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it - // will provide its own version of the code. -#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - simdjson_inline void write(uint32_t idx, uint64_t bits); -#else - simdjson_inline void write(uint32_t idx, uint64_t bits) { - // In some instances, the next branch is expensive because it is mispredicted. - // Unfortunately, in other cases, - // it helps tremendously. - if (bits == 0) - return; - - int cnt = static_cast(count_ones(bits)); - -#if SIMDJSON_PREFER_REVERSE_BITS - bits = reverse_bits(bits); -#endif -#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP - static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; -#else - static constexpr const int STEP = 4; -#endif - static constexpr const int STEP_UNTIL = 24; - - write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); - SIMDJSON_IF_CONSTEXPR (STEP_UNTIL < 64) { - if (simdjson_unlikely(STEP_UNTIL < cnt)) { - for (int i=STEP_UNTIL; itail += cnt; - } -#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -}; - -class json_structural_indexer { -public: - /** - * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. - * - * @param partial Setting the partial parameter to true allows the find_structural_bits to - * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If - * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. - */ - template - static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept; - -private: - simdjson_inline json_structural_indexer(uint32_t *structural_indexes); - template - simdjson_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_warn_unused simdjson_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); - - json_scanner scanner{}; - utf8_checker checker{}; - bit_indexer indexer; - uint64_t prev_structurals = 0; - uint64_t unescaped_chars_error = 0; -}; - -simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} - -// Skip the last character if it is partial -simdjson_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { - if (simdjson_unlikely(len < 3)) { - switch (len) { - case 2: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left - return len; - case 1: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - return len; - case 0: - return len; - } - } - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left - return len; -} - -// -// PERF NOTES: -// We pipe 2 inputs through these stages: -// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load -// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. -// 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. -// The output of step 1 depends entirely on this information. These functions don't quite use -// up enough CPU: the second half of the functions is highly serial, only using 1 execution core -// at a time. The second input's scans has some dependency on the first ones finishing it, but -// they can make a lot of progress before they need that information. -// 3. Step 1 does not use enough capacity, so we run some extra stuff while we're waiting for that -// to finish: utf-8 checks and generating the output from the last iteration. -// -// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all -// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough -// workout. -// -template -error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept { - if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } - // We guard the rest of the code so that we can assume that len > 0 throughout. - if (len == 0) { return EMPTY; } - if (is_streaming(partial)) { - len = trim_partial_utf8(buf, len); - // If you end up with an empty window after trimming - // the partial UTF-8 bytes, then chances are good that you - // have an UTF-8 formatting error. - if(len == 0) { return UTF8_ERROR; } - } - buf_block_reader reader(buf, len); - json_structural_indexer indexer(parser.structural_indexes.get()); - - // Read all but the last block - while (reader.has_full_block()) { - indexer.step(reader.full_block(), reader); - } - // Take care of the last block (will always be there unless file is empty which is - // not supposed to happen.) - uint8_t block[STEP_SIZE]; - const size_t remaining = parser._unpadded ? reader.get_remainder(block) : reader.get_remainder_padded(block); - if (simdjson_unlikely(remaining == 0)) { return UNEXPECTED_ERROR; } - indexer.step(block, reader); - return indexer.finish(parser, reader.block_index(), len, partial); -} - -template<> -simdjson_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block); - simd::simd8x64 in_2(block+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1, reader.block_index()); - this->next(in_2, block_2, reader.block_index()+64); - reader.advance(); -} - -template<> -simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block); - json_block block_1 = scanner.next(in_1); - this->next(in_1, block_1, reader.block_index()); - reader.advance(); -} - -simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { - uint64_t unescaped = in.lteq(0x1F); -#if SIMDJSON_UTF8VALIDATION - checker.check_next_input(in); -#endif - indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser - prev_structurals = block.structural_start(); - unescaped_chars_error |= block.non_quote_inside_string(unescaped); -} - -// Drops the unclosed string that ends a json_sequence batch and leaves its -// start in structural_indexes[n_structural_indexes]. The last index is its -// opening quote, unless the quote directly follows an RS: the scanner reads -// RS as a scalar byte, so it indexes the RS and not the quote. Keep the RS. -simdjson_inline void drop_json_sequence_unclosed_string(dom_parser_implementation &parser, size_t len) noexcept { - uint32_t quote = parser.structural_indexes[parser.n_structural_indexes - 1]; - if (parser.buf[quote] == '"') { - parser.n_structural_indexes--; - return; - } - while (quote < len && parser.buf[quote] != '"') { quote++; } - parser.structural_indexes[parser.n_structural_indexes] = quote; -} - -simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial) { - // Write out the final iteration's structurals - indexer.write(uint32_t(idx-64), prev_structurals); - error_code error = scanner.finish(); - // We deliberately break down the next expression so that it is - // human readable. - const bool should_we_exit = is_streaming(partial) ? - ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING - : (error != SUCCESS); // if partial is false, we must have SUCCESS - const bool have_unclosed_string = (error == UNCLOSED_STRING); - parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); - /*** - * The On-Demand API requires special padding. - * - * This is related to https://github.com/simdjson/simdjson/issues/906 - * Basically, we want to make sure that if the parsing continues beyond the last (valid) - * structural character, it quickly stops. - * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. - * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing - * continues, then it must be [,] or }. - * Suppose it is ] or }. We backtrack to the first character, what could it be that would - * not trigger an error? It could be ] or } but no, because you can't start a document that way. - * It can't be a comma, a colon or any simple value. So the only way we could continue is - * if the repeated character is [. But if so, the document must start with [. But if the document - * starts with [, it should end with ]. If we enforce that rule, then we would get - * ][[ which is invalid. - * - * This is illustrated with the test array_iterate_unclosed_error() on the following input: - * R"({ "a": [,,)" - **/ - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final - parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); - parser.structural_indexes[parser.n_structural_indexes + 2] = 0; - parser.next_structural_index = 0; - - // Bail out only once the count and sentinels above are set: - // document_stream::truncated_bytes() reads them even on error. - if (simdjson_unlikely(should_we_exit)) { return error; } - if (unescaped_chars_error) { return UNESCAPED_CHARS; } - - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - return EMPTY; - } - if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { - return UNEXPECTED_ERROR; - } - if (partial == stage1_mode::streaming_partial) { - // If we have an unclosed string, then the last structural - // will be the quote and we want to make sure to omit it. - if(have_unclosed_string) { - parser.n_structural_indexes--; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - // We truncate the input to the end of the last complete document (or zero). - const bool partial_scalar = !have_unclosed_string && ends_with_partial_scalar(parser, len); - const bool unfinished = ends_with_unfinished_document(parser); - auto new_structural_indexes = find_next_document_index(parser, partial_scalar || unfinished); - if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { - // The batch holds a single document, whose brackets balance but which - // cannot be valid JSON: it is not too big, it is invalid. - if (unfinished && find_next_document_index(parser, partial_scalar) != 0) { return TAPE_ERROR; } - if(parser.structural_indexes[0] == 0) { - // If the buffer is partial and we started at index 0 but the document is - // incomplete, it's too big to parse. - return CAPACITY; - } else { - // It is possible that the document could be parsed, we just had a lot - // of white space. - parser.n_structural_indexes = 0; - return EMPTY; - } - } - parser.n_structural_indexes = new_structural_indexes; - } else if (partial == stage1_mode::streaming_final) { - if(have_unclosed_string) { parser.n_structural_indexes--; } - // We truncate the input to the end of the last complete document (or zero). - // Because partial == stage1_mode::streaming_final, it means that we may - // silently ignore trailing garbage. Though it sounds bad, we do it - // deliberately because many people who have streams of JSON documents - // will truncate them for processing. E.g., imagine that you are uncompressing - // the data from a size file or receiving it in chunks from the network. You - // may not know where exactly the last document will be. Meanwhile the - // document_stream instances allow people to know the JSON documents they are - // parsing (see the iterator.source() method). - parser.n_structural_indexes = find_next_document_index(parser); - // We store the initial n_structural_indexes so that the client can see - // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, - // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, - // otherwise, it will copy some prior index. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - // This next line is critical, do not change it unless you understand what you are - // doing. - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - // We tolerate an unclosed string at the very end of the stream. Indeed, users - // often load their data in bulk without being careful and they want us to ignore - // the trailing garbage. - return EMPTY; - } - } else if (partial == stage1_mode::json_sequence_partial) { - // RFC 7464: use RS positions for batch boundaries - // A discarded unclosed string also caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - // The next batch starts at an RS, which the On-Demand API never reads as a - // string, a container or a separator. But an invalid last document may make - // it skip that RS as a value: repeat it, so that the index past it is not - // the next record, which can be a string that does not close in this batch. - parser.structural_indexes[parser.n_structural_indexes + 1] = next_batch_start; - } else if (partial == stage1_mode::json_sequence_final) { - // RFC 7464: final batch, last document extends to EOF - // As above: a discarded unclosed string caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); - // The filter compacted structural_indexes in place and restored the EOF - // sentinel past the compacted end, so the copy below is either the start - // of a truncated document or len, as in streaming_final. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } else if (partial == stage1_mode::comma_delimited_partial) { - // Comma-delimited: filter root-level commas, use comma positions for batch boundaries - if(have_unclosed_string) { - parser.n_structural_indexes--; - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - } else if (partial == stage1_mode::comma_delimited_final) { - // Comma-delimited: final batch, last document extends to EOF - if(have_unclosed_string) { parser.n_structural_indexes--; } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } - checker.check_eof(); - return checker.errors(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. -#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H -/* end file generic/stage1/json_structural_indexer.h for ppc64 */ -/* including generic/stage1/utf8_validator.h for ppc64: #include */ -/* begin file generic/stage1/utf8_validator.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage1 { - -/** - * Validates that the string is actual UTF-8. - */ -template -bool generic_validate_utf8(const uint8_t * input, size_t length) { - checker c{}; - buf_block_reader<64> reader(input, length); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - c.check_next_input(in); - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - c.check_next_input(in); - reader.advance(); - c.check_eof(); - return c.errors() == error_code::SUCCESS; -} - -bool generic_validate_utf8(const char * input, size_t length) { - return generic_validate_utf8(reinterpret_cast(input),length); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H -/* end file generic/stage1/utf8_validator.h for ppc64 */ -/* end file generic/stage1/amalgamated.h for ppc64 */ -/* including generic/stage2/amalgamated.h for ppc64: #include */ -/* begin file generic/stage2/amalgamated.h for ppc64 */ -// Stuff other things depend on -/* including generic/stage2/base.h for ppc64: #include */ -/* begin file generic/stage2/base.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage2 { - -class json_iterator; -struct tape_builder; -struct tape_writer; - -} // namespace stage2 -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H -/* end file generic/stage2/base.h for ppc64 */ -/* including generic/stage2/tape_writer.h for ppc64: #include */ -/* begin file generic/stage2/tape_writer.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage2 { - -struct tape_writer { - /** The next place to write to tape */ - uint64_t *next_tape_loc; - - /** Write a signed 64-bit value to tape. */ - simdjson_inline void append_s64(int64_t value) noexcept; - - /** Write an unsigned 64-bit value to tape. */ - simdjson_inline void append_u64(uint64_t value) noexcept; - - /** Write a double value to tape. */ - simdjson_inline void append_double(double value) noexcept; - - /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ - simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; - - /** - * Append a tape entry (an 8-bit type,and 56 bits worth of value). - */ - simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; - - /** - * Skip the current tape entry without writing. - * - * Used to skip the start of the container, since we'll come back later to fill it in when the - * container ends. - */ - simdjson_inline void skip() noexcept; - - /** - * Skip the number of tape entries necessary to write a large u64 or i64. - */ - simdjson_inline void skip_large_integer() noexcept; - - /** - * Skip the number of tape entries necessary to write a double. - */ - simdjson_inline void skip_double() noexcept; - - /** - * Write a value to a known location on tape. - * - * Used to go back and write out the start of a container after the container ends. - */ - simdjson_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; - -private: - /** - * Append both the tape entry, and a supplementary value following it. Used for types that need - * all 64 bits, such as double and uint64_t. - */ - template - simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; -}; // struct tape_writer - -simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { - append2(0, value, internal::tape_type::INT64); -} - -simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { - append(0, internal::tape_type::UINT64); - *next_tape_loc = value; - next_tape_loc++; -} - -/** Write a double value to tape. */ -simdjson_inline void tape_writer::append_double(double value) noexcept { - append2(0, value, internal::tape_type::DOUBLE); -} - -simdjson_inline void tape_writer::skip() noexcept { - next_tape_loc++; -} - -simdjson_inline void tape_writer::skip_large_integer() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::skip_double() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { - *next_tape_loc = val | ((uint64_t(char(t))) << 56); - next_tape_loc++; -} - -template -simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { - append(val, t); - static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); - memcpy(next_tape_loc, &val2, sizeof(val2)); - next_tape_loc++; -} - -simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { - tape_loc = val | ((uint64_t(char(t))) << 56); -} - -simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { - // Write to string buffer: [4-byte LE length][digits][null] - uint32_t str_len = uint32_t(len); - memcpy(string_buf, &str_len, sizeof(uint32_t)); - memcpy(string_buf + sizeof(uint32_t), src, len); - string_buf[sizeof(uint32_t) + len] = 0; - // Tape entry: offset into string buffer - // The caller must set the offset relative to doc.string_buf base - append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches - string_buf += sizeof(uint32_t) + len + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H -/* end file generic/stage2/tape_writer.h for ppc64 */ -/* including generic/stage2/logger.h for ppc64: #include */ -/* begin file generic/stage2/logger.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - - -// This is for an internal-only stage 2 specific logger. -// Set LOG_ENABLED = true to log what stage 2 is doing! -namespace simdjson { -namespace ppc64 { -namespace { -namespace logger { - - static constexpr const char * DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; - -#if SIMDJSON_VERBOSE_LOGGING - static constexpr const bool LOG_ENABLED = true; -#else - static constexpr const bool LOG_ENABLED = false; -#endif - static constexpr const int LOG_EVENT_LEN = 20; - static constexpr const int LOG_BUFFER_LEN = 30; - static constexpr const int LOG_SMALL_BUFFER_LEN = 10; - static constexpr const int LOG_INDEX_LEN = 5; - - static int log_depth; // Not threadsafe. Log only. - - // Helper to turn unprintable or newline characters into spaces - static simdjson_inline char printable_char(char c) { - if (c >= 0x20) { - return c; - } else { - return ' '; - } - } - - // Print the header and set up log_start - static simdjson_inline void log_start() { - if (LOG_ENABLED) { - log_depth = 0; - printf("\n"); - printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); - printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN+2, DASHES, LOG_BUFFER_LEN+2, DASHES, LOG_SMALL_BUFFER_LEN+2, DASHES, 5+2, DASHES); - } - } - - simdjson_unused static simdjson_inline void log_string(const char *message) { - if (LOG_ENABLED) { - printf("%s\n", message); - } - } - - // Logs a single line from the stage 2 DOM parser - template - static simdjson_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { - if (LOG_ENABLED) { - printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); - auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; - auto next_index = structurals.next_structural; - auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); - auto next = &structurals.buf[*next_index]; - { - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i=0;i */ -/* begin file generic/stage2/json_iterator.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage2 { - -class json_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - uint32_t depth{0}; - - /** - * Walk the JSON document. - * - * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as - * the first parameter; some callbacks have other parameters as well: - * - * - visit_document_start() - at the beginning. - * - visit_document_end() - at the end (if things were successful). - * - * - visit_array_start() - at the start `[` of a non-empty array. - * - visit_array_end() - at the end `]` of a non-empty array. - * - visit_empty_array() - when an empty array is encountered. - * - * - visit_object_end() - at the start `]` of a non-empty object. - * - visit_object_start() - at the end `]` of a non-empty object. - * - visit_empty_object() - when an empty object is encountered. - * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is - * guaranteed to point at the first quote of the string (`"key"`). - * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. - * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. - * - * - increment_count(iter) - each time a value is found in an array or object. - */ - template - simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; - - /** - * Create an iterator capable of walking a JSON document. - * - * The document must have already passed through stage 1. - */ - simdjson_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); - - /** - * Look at the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *peek(const uint32_t *batch_end = nullptr) const noexcept; - /** - * Advance to the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *advance(const uint32_t *batch_end = nullptr) noexcept; - /** - * Get the remaining length of the document, from the start of the current token. - */ - simdjson_inline size_t remaining_len() const noexcept; - /** - * Check if we are at the end of the document. - * - * If this is true, there are no more tokens. - */ - simdjson_inline bool at_eof() const noexcept; - /** - * Check if we are at the beginning of the document. - */ - simdjson_inline bool at_beginning() const noexcept; - simdjson_inline uint8_t last_structural() const noexcept; - - /** - * Log that a value has been found. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_value(const char *type) const noexcept; - /** - * Log the start of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_start_value(const char *type) const noexcept; - /** - * Log the end of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_end_value(const char *type) const noexcept; - /** - * Log an error. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_error(const char *error) const noexcept; - - template - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; - template - simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; -}; - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { - logger::log_start(); - - // - // Start the document - // - if (at_eof()) { return EMPTY; } - // Only used when BOUNDED (see advance()); a local, so that it can stay in a - // register, and vanishes from the other walks. - const uint32_t *const batch_end = &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - log_start_value("document"); - SIMDJSON_TRY( visitor.visit_document_start(*this) ); - - // - // Read first value - // - { - auto value = advance(batch_end); - - // Make sure the outer object or array is closed before continuing; otherwise, there are ways we - // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 - if (!STREAMING) { - switch (*value) { - case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; - case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; - } - } - - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; - } - } - goto document_end; - -// -// Object parser states -// -object_begin: - log_start_value("object"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = false; - SIMDJSON_TRY( visitor.visit_object_start(*this) ); - - { - auto key = advance(batch_end); - if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.increment_count(*this) ); - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - -object_field: - if (simdjson_unlikely( (*advance(batch_end) != ':') )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -object_continue: - switch (*advance(batch_end)) { - case ',': - SIMDJSON_TRY( visitor.increment_count(*this) ); - { - auto key = advance(batch_end); - if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - goto object_field; - case '}': log_end_value("object"); SIMDJSON_TRY( visitor.visit_object_end(*this) ); goto scope_end; - default: log_error("No comma between object fields"); return TAPE_ERROR; - } - -scope_end: - depth--; - if (depth == 0) { goto document_end; } - if (dom_parser.is_array[depth]) { goto array_continue; } - goto object_continue; - -// -// Array parser states -// -array_begin: - log_start_value("array"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = true; - SIMDJSON_TRY( visitor.visit_array_start(*this) ); - SIMDJSON_TRY( visitor.increment_count(*this) ); - -array_value: - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -array_continue: - switch (*advance(batch_end)) { - case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; - case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; - default: log_error("Missing comma between array values"); return TAPE_ERROR; - } - -document_end: - log_end_value("document"); - SIMDJSON_TRY( visitor.visit_document_end(*this) ); - - dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); - - // If we didn't make it to the end, it's an error - if ( !STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes ) { - log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); - return TAPE_ERROR; - } - - return SUCCESS; - -} // walk_document() - -simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { -} - -// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires -// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). -// -// When BOUNDED (see dom_parser_implementation::_bounded_batch), a document may -// not run past batch_end, one past the last structural index of the batch: -// past it, we return the same dummy, and walk_document fails cleanly. -template -simdjson_inline const uint8_t *json_iterator::peek(const uint32_t *batch_end) const noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -template -simdjson_inline const uint8_t *json_iterator::advance(const uint32_t *batch_end) noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural++); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -simdjson_inline size_t json_iterator::remaining_len() const noexcept { - return dom_parser.len - *(next_structural-1); -} - -simdjson_inline bool json_iterator::at_eof() const noexcept { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; -} -simdjson_inline bool json_iterator::at_beginning() const noexcept { - return next_structural == dom_parser.structural_indexes.get(); -} -simdjson_inline uint8_t json_iterator::last_structural() const noexcept { - return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; -} - -simdjson_inline void json_iterator::log_value(const char *type) const noexcept { - logger::log_line(*this, "", type, ""); -} - -simdjson_inline void json_iterator::log_start_value(const char *type) const noexcept { - logger::log_line(*this, "+", type, ""); - if (logger::LOG_ENABLED) { logger::log_depth++; } -} - -simdjson_inline void json_iterator::log_end_value(const char *type) const noexcept { - if (logger::LOG_ENABLED) { logger::log_depth--; } - logger::log_line(*this, "-", type, ""); -} - -simdjson_inline void json_iterator::log_error(const char *error) const noexcept { - logger::log_line(*this, "", "ERROR", error); -} - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { - switch (*value) { - case '"': return visitor.visit_root_string(*this, value); - case 't': return visitor.visit_root_true_atom(*this, value); - case 'f': return visitor.visit_root_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_root_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_root_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_root_inf_atom(*this, value); -#else - case 'n': return visitor.visit_root_null_atom(*this, value); -#endif - case '-': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return visitor.visit_root_number(*this, value); - default: - log_error("Document starts with a non-value character"); - return TAPE_ERROR; - } -} -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { - // Use the fact that most scalars are going to be either strings or numbers. - if(*value == '"') { - return visitor.visit_string(*this, value); - } else if (((*value - '0') < 10) || (*value == '-')) { - return visitor.visit_number(*this, value); - } - // true, false, null are uncommon. - switch (*value) { - case 't': return visitor.visit_true_atom(*this, value); - case 'f': return visitor.visit_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_inf_atom(*this, value); -#else - case 'n': return visitor.visit_null_atom(*this, value); -#endif - default: - log_error("Non-value found when value was expected!"); - return TAPE_ERROR; - } -} - -} // namespace stage2 -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H -/* end file generic/stage2/json_iterator.h for ppc64 */ -/* including generic/stage2/stringparsing.h for ppc64: #include */ -/* begin file generic/stage2/stringparsing.h for ppc64 */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace ppc64 { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - // Decode adjacent Unicode escapes without returning to the - // quote-and-backslash scanner between code points. - do { - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } while (src[0] == '\\' && src[1] == 'u'); - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -/** - * Bounds-safe variant of parse_string for input buffers that are NOT padded to - * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input - * byte (buf + len). It behaves exactly like parse_string while we are at least - * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays - * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few - * remaining bytes into a space-padded scratch buffer and finishes with the - * regular parse_string. This keeps the delicate escape/Unicode handling in one - * place (the proven parse_string) rather than duplicating it. - * - * Correctness relies on stage 1 having validated the string, i.e. there is an - * unescaped closing quote within [src, buf_end); that quote is therefore inside - * the copied scratch, so parse_string finds it without running off the scratch. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { - // Far from the end: identical to parse_string's loop. The guard uses - // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past - // buf_end; escape/Unicode look-aheads read within the string (before the - // closing quote, which is < buf_end), so they are in bounds here too. - // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: - // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate - // following bytes (+6 from the '\'), then (if it sees \u) another - // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the - // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the - // chunk start. The +12 margin ensures that even on kernels where - // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays - // in-bounds. The scratch fallback (3*PAD) is already safe. - while (src + SIMDJSON_PADDING + 12 <= buf_end) { - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - if (bs_quote.has_quote_first()) { - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - if (escape_char == 'u') { - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - // Within the final SIMDJSON_PADDING bytes: copy what remains into a - // space-padded scratch (spaces are neither quote nor backslash, so they do not - // disturb matching) and let the regular parser finish from there. The closing - // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in - // the chunk starting at some offset <= remaining and reads at most - // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. - // We size at 3x for a comfortable margin (the unicode look-ahead reads a few - // extra bytes past an escape). - uint8_t scratch[SIMDJSON_PADDING * 3]; - const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING - std::memset(scratch, ' ', sizeof(scratch)); - std::memcpy(scratch, src, remaining); - return parse_string(scratch, dst, allow_replacement); -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for ppc64 */ -/* including generic/stage2/tape_builder.h for ppc64: #include */ -/* begin file generic/stage2/tape_builder.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage2 { - -template -struct tape_builder_impl { - /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; - /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator &iter) noexcept; - - /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator &iter) noexcept; - /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator &iter) noexcept; - /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator &iter) noexcept; - - /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator &iter) noexcept; - /** - * Called when a key in a field is encountered. - * - * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array - * will be called after this with the field value. - */ - simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; - /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator &iter) noexcept; - /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator &iter) noexcept; - - /** - * Called when a string, number, boolean or null is found. - */ - simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; - /** - * Called when a string, number, boolean or null is found at the top level of a document (i.e. - * when there is no array or object and the entire document is a single string, number, boolean or - * null. - * - * This is separate from primitive() because simdjson's normal primitive parsing routines assume - * there is at least one more token after the value, which is only true in an array or object. - */ - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - -#if SIMDJSON_ENABLE_NAN_INF - simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, - // this returns a tape error on failure. - simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; -#endif - - /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; - - /** Next location to write to tape */ - tape_writer tape; -public: - simdjson_inline tape_builder_impl(dom::document &doc) noexcept; -private: - /** Next write location in the string buf for stage 2 parsing */ - uint8_t *current_string_buf_loc; - - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - simdjson_inline void start_container(json_iterator &iter) noexcept; - simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder_impl - -// Thin, non-templated entry so each architecture's stage2() keeps calling -// tape_builder::parse_document unchanged. It chooses the bounds-safe -// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the -// choice is a compile-time constant inside the walk: the padded path carries no -// extra branch or load (see tape_builder_impl::visit_string). -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } - - // The stage 2 entry points of dom::document_stream, which picks one per batch - // (see internal::dom_parser_implementation::stage2_next_function). Out of - // line, so that stage2_next() can share the code. - simdjson_warn_unused static simdjson_never_inline error_code parse_next_document( - internal::dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - return parse_document(static_cast(dom_parser), doc); - } - - // A document of a stream batch that stage 2 must bound to its structural - // indexes (see internal::dom_parser_implementation::_bounded_batch). - simdjson_warn_unused static simdjson_never_inline error_code parse_next_bounded_document( - internal::dom_parser_implementation &_dom_parser, dom::document &doc) noexcept { - dom_parser_implementation &dom_parser = static_cast(_dom_parser); - dom_parser.doc = &doc; - json_iterator iter(dom_parser, dom_parser.next_structural_index); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } -}; - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_root_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { - constexpr uint32_t start_tape_index = 0; - tape.append(start_tape_index, internal::tape_type::ROOT); - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { - return visit_string(iter, key, true); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 - return SUCCESS; -} - -template -simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { - iter.log_value(key ? "key" : "string"); - uint8_t *dst = on_start_string(iter); - // We do not allow replacement when the escape characters are invalid. - // UNPADDED is a compile-time constant chosen once per document by - // tape_builder::parse_document, so the padded build instantiates only the - // plain parse_string call below -- no runtime branch and no flag load. - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); - } else { - dst = stringparsing::parse_string(value+1, dst, false); - } - if (dst == nullptr) { - iter.log_error("Invalid escape in string"); - return STRING_ERROR; - } - on_end_string(dst); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { - return visit_string(iter, value); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("number"); - const uint8_t *num = value; - std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - // numberparsing reads ahead in 8-byte blocks for floats - // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a - // number whose digits reach the final bytes of an unpadded buffer would read - // past it. *(next_structural) is the offset of the token following this - // number, hence an upper bound on where the digits end; when that is within - // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail - // (mirroring visit_root_number). This fires only for numbers near the end. - if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { - const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document - copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, rl); - std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); - num = copy.get(); - } - } - error_code err = numberparsing::parse_number(num, tape); - if (simdjson_unlikely(err == BIGINT_ERROR && - iter.dom_parser._number_as_string)) { - // Write big integer to string buffer using the same format as strings. - // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = num; - if (*p == '-') p++; - while (numberparsing::is_digit(*p)) p++; - // The digit run must be terminated by a structural or whitespace character; otherwise the - // token is malformed (e.g. "123456789123456789123x"). - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - num); - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); - uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, num, len); - dst += len; - on_end_string(dst); - return SUCCESS; - } - return err; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { - // - // We need to make a copy to make sure that the string is space terminated. - // This is not about padding the input, which should already padded up - // to len + SIMDJSON_PADDING. However, we have no control at this stage - // on how the padding was done. What if the input string was padded with nulls? - // It is quite common for an input string to have an extra null character (C string). - // We do not want to allow 9\0 (where \0 is the null character) inside a JSON - // document, but the string "9\0" by itself is fine. So we make a copy and - // pad the input with spaces when we know that there is just one input element. - // This copy is relatively expensive, but it will almost never be called in - // practice unless you are in the strange scenario where you have many JSON - // documents made of single atoms. - // - // In a stream, the input goes on with other documents: copy up to the next - // structural only, not to the end of the batch. - const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); - std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, len); - std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); - error_code error = visit_number(iter, copy.get()); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated - // token at the very end of an unpadded buffer would over-read. Use the - // length-aware form there (the root variant already does this). - const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) - : atomparsing::is_valid_true_atom(value); - if (!ok) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) - : atomparsing::is_valid_false_atom(value); - if (!ok) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) - : atomparsing::is_valid_null_atom(value); - if (!ok) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -#if SIMDJSON_ENABLE_NAN_INF -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - // For unpadded input use the length-aware validator so the 'infinity'-style - // 8-byte compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) - : atomparsing::is_valid_nan_atom(value); - if (!ok) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. - // For unpadded input use the length-aware validator so the 'infinity' 8-byte - // compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) - : atomparsing::is_valid_inf_atom(value); - if (!ok) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure - if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -// private: - -template -simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { - return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - auto start_index = next_tape_index(iter); - tape.append(start_index+2, start); - tape.append(start_index, end); - return SUCCESS; -} - -template -simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); - iter.dom_parser.open_containers[iter.depth].count = 0; - tape.skip(); // We don't actually *write* the start element until the end. -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - // Write the ending tape element, pointing at the start location - const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; - tape.append(start_tape_index, end); - // Write the start tape element, pointing at the end location (and including count) - // count can overflow if it exceeds 24 bits... so we saturate - // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). - const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; - const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); - return SUCCESS; -} - -template -simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { - // we advance the point, accounting for the fact that we have a NULL termination - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); - return current_string_buf_loc + sizeof(uint32_t); -} - -template -simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { - uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); - // TODO check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); - // NULL termination is still handy if you expect all your strings to - // be NULL terminated? It comes at a small cost - *dst = 0; - current_string_buf_loc = dst + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H -/* end file generic/stage2/tape_builder.h for ppc64 */ -/* end file generic/stage2/amalgamated.h for ppc64 */ - -// -// Stage 1 -// -namespace simdjson { -namespace ppc64 { - -simdjson_warn_unused error_code implementation::create_dom_parser_implementation( - size_t capacity, - size_t max_depth, - std::unique_ptr& dst -) const noexcept { - dst.reset( new (std::nothrow) dom_parser_implementation() ); - if (!dst) { return MEMALLOC; } - if (auto err = dst->set_capacity(capacity)) - return err; - if (auto err = dst->set_max_depth(max_depth)) - return err; - return SUCCESS; -} - -namespace { - -using namespace simd; - -simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - const simd8 table1(16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0); - const simd8 table2(8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); - - simd8x64 v( - (in.chunks[0] & 0xf).lookup_16(table1) & (in.chunks[0].shr<4>()).lookup_16(table2), - (in.chunks[1] & 0xf).lookup_16(table1) & (in.chunks[1].shr<4>()).lookup_16(table2), - (in.chunks[2] & 0xf).lookup_16(table1) & (in.chunks[2].shr<4>()).lookup_16(table2), - (in.chunks[3] & 0xf).lookup_16(table1) & (in.chunks[3].shr<4>()).lookup_16(table2) - ); - - uint64_t op = simd8x64( - v.chunks[0].any_bits_set(0x7), - v.chunks[1].any_bits_set(0x7), - v.chunks[2].any_bits_set(0x7), - v.chunks[3].any_bits_set(0x7) - ).to_bitmask(); - - uint64_t whitespace = simd8x64( - v.chunks[0].any_bits_set(0x18), - v.chunks[1].any_bits_set(0x18), - v.chunks[2].any_bits_set(0x18), - v.chunks[3].any_bits_set(0x18) - ).to_bitmask(); - - return { whitespace, op }; -} - -simdjson_inline bool is_ascii(const simd8x64& input) { - // careful: 0x80 is not ascii. - return input.reduce_or().saturating_sub(0x7fu).bits_not_set_anywhere(); -} - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be >= 0x80 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be >= 0x80 - return is_third_byte | is_fourth_byte; -} - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -// -// Stage 2 -// - -// -// Implementation-specific overrides -// -namespace simdjson { -namespace ppc64 { - -simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { - return ppc64::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, stage1_mode streaming) noexcept { - this->buf = _buf; - this->len = _len; - return ppc64::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); -} - -simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return ppc64::stage1::generic_validate_utf8(buf,len); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_next_document(*this, _doc); -} - -internal::dom_parser_implementation::stage2_next_function_t dom_parser_implementation::stage2_next_function(bool bounded) const noexcept { - return bounded ? stage2::tape_builder::parse_next_bounded_document : stage2::tape_builder::parse_next_document; -} - -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { - return ppc64::stringparsing::parse_string(src, dst, replacement_char); -} - -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { - return ppc64::stringparsing::parse_wobbly_string(src, dst); -} - -simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { - auto error = stage1(_buf, _len, stage1_mode::regular); - if (error) { return error; } - return stage2(_doc); -} - -} // namespace ppc64 -} // namespace simdjson - -/* including simdjson/ppc64/end.h: #include */ -/* begin file simdjson/ppc64/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT -/* undefining SIMDJSON_IMPLEMENTATION from "ppc64" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/ppc64/end.h */ - -#endif // SIMDJSON_SRC_PPC64_CPP -/* end file ppc64.cpp */ -#endif -#if SIMDJSON_IMPLEMENTATION_WESTMERE -/* including westmere.cpp: #include */ -/* begin file westmere.cpp */ -#ifndef SIMDJSON_SRC_WESTMERE_CPP -#define SIMDJSON_SRC_WESTMERE_CPP - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* including simdjson/westmere.h: #include */ -/* begin file simdjson/westmere.h */ -#ifndef SIMDJSON_WESTMERE_H -#define SIMDJSON_WESTMERE_H - -/* including simdjson/westmere/begin.h: #include "simdjson/westmere/begin.h" */ -/* begin file simdjson/westmere/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "westmere" */ -#define SIMDJSON_IMPLEMENTATION westmere -/* including simdjson/westmere/base.h: #include "simdjson/westmere/base.h" */ -/* begin file simdjson/westmere/base.h */ -#ifndef SIMDJSON_WESTMERE_BASE_H -#define SIMDJSON_WESTMERE_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE -namespace simdjson { -/** - * Implementation for Westmere (Intel SSE4.2). - */ -namespace westmere { - -class implementation; - -namespace { -namespace simd { - -template struct simd8; -template struct simd8x64; - -} // namespace simd -} // unnamed namespace - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BASE_H -/* end file simdjson/westmere/base.h */ -/* including simdjson/westmere/intrinsics.h: #include "simdjson/westmere/intrinsics.h" */ -/* begin file simdjson/westmere/intrinsics.h */ -#ifndef SIMDJSON_WESTMERE_INTRINSICS_H -#define SIMDJSON_WESTMERE_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - */ -#include // for _mm_alignr_epi8 -#include // for _mm_clmulepi64_si128 -#endif - -static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); - -#endif // SIMDJSON_WESTMERE_INTRINSICS_H -/* end file simdjson/westmere/intrinsics.h */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_WESTMERE -SIMDJSON_TARGET_REGION("sse4.2,pclmul,popcnt") -#endif - -/* including simdjson/westmere/bitmanipulation.h: #include "simdjson/westmere/bitmanipulation.h" */ -/* begin file simdjson/westmere/bitmanipulation.h */ -#ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H -#define SIMDJSON_WESTMERE_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO -} - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num);// Visual Studio wants two underscores -} -#else -simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); -} -#endif - - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BITMANIPULATION_H -/* end file simdjson/westmere/bitmanipulation.h */ -/* including simdjson/westmere/bitmask.h: #include "simdjson/westmere/bitmask.h" */ -/* begin file simdjson/westmere/bitmask.h */ -#ifndef SIMDJSON_WESTMERE_BITMASK_H -#define SIMDJSON_WESTMERE_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { - // There should be no such thing with a processing supporting avx2 - // but not clmul. - __m128i all_ones = _mm_set1_epi8('\xFF'); - __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); - return _mm_cvtsi128_si64(result); -} - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BITMASK_H -/* end file simdjson/westmere/bitmask.h */ -/* including simdjson/westmere/numberparsing_defs.h: #include "simdjson/westmere/numberparsing_defs.h" */ -/* begin file simdjson/westmere/numberparsing_defs.h */ -#ifndef SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H -#define SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H - -/* including simdjson/westmere/base.h: #include "simdjson/westmere/base.h" */ -/* begin file simdjson/westmere/base.h */ -#ifndef SIMDJSON_WESTMERE_BASE_H -#define SIMDJSON_WESTMERE_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE -namespace simdjson { -/** - * Implementation for Westmere (Intel SSE4.2). - */ -namespace westmere { - -class implementation; - -namespace { -namespace simd { - -template struct simd8; -template struct simd8x64; - -} // namespace simd -} // unnamed namespace - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BASE_H -/* end file simdjson/westmere/base.h */ -/* including simdjson/westmere/intrinsics.h: #include "simdjson/westmere/intrinsics.h" */ -/* begin file simdjson/westmere/intrinsics.h */ -#ifndef SIMDJSON_WESTMERE_INTRINSICS_H -#define SIMDJSON_WESTMERE_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - */ -#include // for _mm_alignr_epi8 -#include // for _mm_clmulepi64_si128 -#endif - -static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); - -#endif // SIMDJSON_WESTMERE_INTRINSICS_H -/* end file simdjson/westmere/intrinsics.h */ - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace numberparsing { - -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - // this actually computes *16* values so we are being wasteful. - const __m128i ascii0 = _mm_set1_epi8('0'); - const __m128i mul_1_10 = - _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); - const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); - const __m128i mul_1_10000 = - _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); - const __m128i input = _mm_sub_epi8( - _mm_loadu_si128(reinterpret_cast(chars)), ascii0); - const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); - const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); - const __m128i t3 = _mm_packus_epi32(t2, t2); - const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); - return _mm_cvtsi128_si32( - t4); // only captures the sum of the first 8 digits, drop the rest -} - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace westmere -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H -/* end file simdjson/westmere/numberparsing_defs.h */ -/* including simdjson/westmere/simd.h: #include "simdjson/westmere/simd.h" */ -/* begin file simdjson/westmere/simd.h */ -#ifndef SIMDJSON_WESTMERE_SIMD_H -#define SIMDJSON_WESTMERE_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace simd { - - template - struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{__m128i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i&() const { return this->value; } - simdjson_inline operator __m128i&() { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - template> - struct base8: base> { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return _mm_alignr_epi8(*this, prev_chunk, 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } - simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } - static simdjson_inline simd8 load(const T values[16]) { - return _mm_loadu_si128(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - __m128i shufmask = _mm_set_epi64x(thintable_epi8[mask2], thintable_epi8[mask1]); - // we increment by 0x08 the second half of the mask - shufmask = - _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); - // this is the version "nearly pruned" - __m128i pruned = _mm_shuffle_epi8(*this, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - __m128i compactmask = - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8)); - __m128i answer = _mm_shuffle_epi8(pruned, compactmask); - _mm_storeu_si128(reinterpret_cast<__m128i *>(output), answer); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - this->chunks[2].store(ptr+sizeof(simd8)*2); - this->chunks[3].store(ptr+sizeof(simd8)*3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - this->chunks[0].compress(uint16_t(mask), output); - this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); - this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); - this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); - return 64 - count_ones(mask); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask() ); - uint64_t r1 = this->chunks[1].to_bitmask() ; - uint64_t r2 = this->chunks[2].to_bitmask() ; - uint64_t r3 = this->chunks[3].to_bitmask() ; - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_SIMD_INPUT_H -/* end file simdjson/westmere/simd.h */ -/* including simdjson/westmere/stringparsing_defs.h: #include "simdjson/westmere/stringparsing_defs.h" */ -/* begin file simdjson/westmere/stringparsing_defs.h */ -#ifndef SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H -#define SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H - -/* including simdjson/westmere/bitmanipulation.h: #include "simdjson/westmere/bitmanipulation.h" */ -/* begin file simdjson/westmere/bitmanipulation.h */ -#ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H -#define SIMDJSON_WESTMERE_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO -} - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num);// Visual Studio wants two underscores -} -#else -simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); -} -#endif - - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BITMANIPULATION_H -/* end file simdjson/westmere/bitmanipulation.h */ -/* including simdjson/westmere/simd.h: #include "simdjson/westmere/simd.h" */ -/* begin file simdjson/westmere/simd.h */ -#ifndef SIMDJSON_WESTMERE_SIMD_H -#define SIMDJSON_WESTMERE_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace simd { - - template - struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{__m128i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i&() const { return this->value; } - simdjson_inline operator __m128i&() { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - template> - struct base8: base> { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return _mm_alignr_epi8(*this, prev_chunk, 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } - simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } - static simdjson_inline simd8 load(const T values[16]) { - return _mm_loadu_si128(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - __m128i shufmask = _mm_set_epi64x(thintable_epi8[mask2], thintable_epi8[mask1]); - // we increment by 0x08 the second half of the mask - shufmask = - _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); - // this is the version "nearly pruned" - __m128i pruned = _mm_shuffle_epi8(*this, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - __m128i compactmask = - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8)); - __m128i answer = _mm_shuffle_epi8(pruned, compactmask); - _mm_storeu_si128(reinterpret_cast<__m128i *>(output), answer); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - this->chunks[2].store(ptr+sizeof(simd8)*2); - this->chunks[3].store(ptr+sizeof(simd8)*3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - this->chunks[0].compress(uint16_t(mask), output); - this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); - this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); - this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); - return 64 - count_ones(mask); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask() ); - uint64_t r1 = this->chunks[1].to_bitmask() ; - uint64_t r2 = this->chunks[2].to_bitmask() ; - uint64_t r3 = this->chunks[3].to_bitmask() ; - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_SIMD_INPUT_H -/* end file simdjson/westmere/simd.h */ - -namespace simdjson { -namespace westmere { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + 16); - v0.store(dst); - v1.store(dst + 16); - uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; -} - - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H -/* end file simdjson/westmere/stringparsing_defs.h */ -/* end file simdjson/westmere/begin.h */ -/* including simdjson/generic/amalgamated.h for westmere: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for westmere */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for westmere: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for westmere */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for westmere */ -/* including simdjson/generic/jsoncharutils.h for westmere: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for westmere */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for westmere */ -/* including simdjson/generic/atomparsing.h for westmere: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for westmere */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace westmere { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for westmere */ -/* including simdjson/generic/dom_parser_implementation.h for westmere: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for westmere */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace westmere -} // namespace simdjson - -namespace simdjson { -namespace westmere { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for westmere */ -/* including simdjson/generic/implementation_simdjson_result_base.h for westmere: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for westmere */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for westmere */ -/* including simdjson/generic/numberparsing.h for westmere: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for westmere */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace westmere { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for westmere */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for westmere: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for westmere */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for westmere */ -/* end file simdjson/generic/amalgamated.h for westmere */ -/* including simdjson/westmere/end.h: #include "simdjson/westmere/end.h" */ -/* begin file simdjson/westmere/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_WESTMERE -SIMDJSON_UNTARGET_REGION -#endif - -/* undefining SIMDJSON_IMPLEMENTATION from "westmere" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/westmere/end.h */ - -#endif // SIMDJSON_WESTMERE_H -/* end file simdjson/westmere.h */ -/* including simdjson/westmere/implementation.h: #include */ -/* begin file simdjson/westmere/implementation.h */ -#ifndef SIMDJSON_WESTMERE_IMPLEMENTATION_H -#define SIMDJSON_WESTMERE_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE -namespace simdjson { -namespace westmere { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation("westmere", "Intel/AMD SSE4.2", internal::instruction_set::SSE42 | internal::instruction_set::PCLMULQDQ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_IMPLEMENTATION_H -/* end file simdjson/westmere/implementation.h */ - -/* including simdjson/westmere/begin.h: #include */ -/* begin file simdjson/westmere/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "westmere" */ -#define SIMDJSON_IMPLEMENTATION westmere -/* including simdjson/westmere/base.h: #include "simdjson/westmere/base.h" */ -/* begin file simdjson/westmere/base.h */ -#ifndef SIMDJSON_WESTMERE_BASE_H -#define SIMDJSON_WESTMERE_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE -namespace simdjson { -/** - * Implementation for Westmere (Intel SSE4.2). - */ -namespace westmere { - -class implementation; - -namespace { -namespace simd { - -template struct simd8; -template struct simd8x64; - -} // namespace simd -} // unnamed namespace - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BASE_H -/* end file simdjson/westmere/base.h */ -/* including simdjson/westmere/intrinsics.h: #include "simdjson/westmere/intrinsics.h" */ -/* begin file simdjson/westmere/intrinsics.h */ -#ifndef SIMDJSON_WESTMERE_INTRINSICS_H -#define SIMDJSON_WESTMERE_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - */ -#include // for _mm_alignr_epi8 -#include // for _mm_clmulepi64_si128 -#endif - -static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); - -#endif // SIMDJSON_WESTMERE_INTRINSICS_H -/* end file simdjson/westmere/intrinsics.h */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_WESTMERE -SIMDJSON_TARGET_REGION("sse4.2,pclmul,popcnt") -#endif - -/* including simdjson/westmere/bitmanipulation.h: #include "simdjson/westmere/bitmanipulation.h" */ -/* begin file simdjson/westmere/bitmanipulation.h */ -#ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H -#define SIMDJSON_WESTMERE_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO -} - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num);// Visual Studio wants two underscores -} -#else -simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); -} -#endif - - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BITMANIPULATION_H -/* end file simdjson/westmere/bitmanipulation.h */ -/* including simdjson/westmere/bitmask.h: #include "simdjson/westmere/bitmask.h" */ -/* begin file simdjson/westmere/bitmask.h */ -#ifndef SIMDJSON_WESTMERE_BITMASK_H -#define SIMDJSON_WESTMERE_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { - // There should be no such thing with a processing supporting avx2 - // but not clmul. - __m128i all_ones = _mm_set1_epi8('\xFF'); - __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); - return _mm_cvtsi128_si64(result); -} - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BITMASK_H -/* end file simdjson/westmere/bitmask.h */ -/* including simdjson/westmere/numberparsing_defs.h: #include "simdjson/westmere/numberparsing_defs.h" */ -/* begin file simdjson/westmere/numberparsing_defs.h */ -#ifndef SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H -#define SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H - -/* including simdjson/westmere/base.h: #include "simdjson/westmere/base.h" */ -/* begin file simdjson/westmere/base.h */ -#ifndef SIMDJSON_WESTMERE_BASE_H -#define SIMDJSON_WESTMERE_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE -namespace simdjson { -/** - * Implementation for Westmere (Intel SSE4.2). - */ -namespace westmere { - -class implementation; - -namespace { -namespace simd { - -template struct simd8; -template struct simd8x64; - -} // namespace simd -} // unnamed namespace - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BASE_H -/* end file simdjson/westmere/base.h */ -/* including simdjson/westmere/intrinsics.h: #include "simdjson/westmere/intrinsics.h" */ -/* begin file simdjson/westmere/intrinsics.h */ -#ifndef SIMDJSON_WESTMERE_INTRINSICS_H -#define SIMDJSON_WESTMERE_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - */ -#include // for _mm_alignr_epi8 -#include // for _mm_clmulepi64_si128 -#endif - -static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); - -#endif // SIMDJSON_WESTMERE_INTRINSICS_H -/* end file simdjson/westmere/intrinsics.h */ - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace numberparsing { - -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - // this actually computes *16* values so we are being wasteful. - const __m128i ascii0 = _mm_set1_epi8('0'); - const __m128i mul_1_10 = - _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); - const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); - const __m128i mul_1_10000 = - _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); - const __m128i input = _mm_sub_epi8( - _mm_loadu_si128(reinterpret_cast(chars)), ascii0); - const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); - const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); - const __m128i t3 = _mm_packus_epi32(t2, t2); - const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); - return _mm_cvtsi128_si32( - t4); // only captures the sum of the first 8 digits, drop the rest -} - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace westmere -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H -/* end file simdjson/westmere/numberparsing_defs.h */ -/* including simdjson/westmere/simd.h: #include "simdjson/westmere/simd.h" */ -/* begin file simdjson/westmere/simd.h */ -#ifndef SIMDJSON_WESTMERE_SIMD_H -#define SIMDJSON_WESTMERE_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace simd { - - template - struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{__m128i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i&() const { return this->value; } - simdjson_inline operator __m128i&() { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - template> - struct base8: base> { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return _mm_alignr_epi8(*this, prev_chunk, 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } - simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } - static simdjson_inline simd8 load(const T values[16]) { - return _mm_loadu_si128(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - __m128i shufmask = _mm_set_epi64x(thintable_epi8[mask2], thintable_epi8[mask1]); - // we increment by 0x08 the second half of the mask - shufmask = - _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); - // this is the version "nearly pruned" - __m128i pruned = _mm_shuffle_epi8(*this, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - __m128i compactmask = - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8)); - __m128i answer = _mm_shuffle_epi8(pruned, compactmask); - _mm_storeu_si128(reinterpret_cast<__m128i *>(output), answer); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - this->chunks[2].store(ptr+sizeof(simd8)*2); - this->chunks[3].store(ptr+sizeof(simd8)*3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - this->chunks[0].compress(uint16_t(mask), output); - this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); - this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); - this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); - return 64 - count_ones(mask); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask() ); - uint64_t r1 = this->chunks[1].to_bitmask() ; - uint64_t r2 = this->chunks[2].to_bitmask() ; - uint64_t r3 = this->chunks[3].to_bitmask() ; - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_SIMD_INPUT_H -/* end file simdjson/westmere/simd.h */ -/* including simdjson/westmere/stringparsing_defs.h: #include "simdjson/westmere/stringparsing_defs.h" */ -/* begin file simdjson/westmere/stringparsing_defs.h */ -#ifndef SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H -#define SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H - -/* including simdjson/westmere/bitmanipulation.h: #include "simdjson/westmere/bitmanipulation.h" */ -/* begin file simdjson/westmere/bitmanipulation.h */ -#ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H -#define SIMDJSON_WESTMERE_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO -} - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num);// Visual Studio wants two underscores -} -#else -simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); -} -#endif - - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BITMANIPULATION_H -/* end file simdjson/westmere/bitmanipulation.h */ -/* including simdjson/westmere/simd.h: #include "simdjson/westmere/simd.h" */ -/* begin file simdjson/westmere/simd.h */ -#ifndef SIMDJSON_WESTMERE_SIMD_H -#define SIMDJSON_WESTMERE_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace simd { - - template - struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{__m128i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i&() const { return this->value; } - simdjson_inline operator __m128i&() { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - template> - struct base8: base> { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return _mm_alignr_epi8(*this, prev_chunk, 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } - simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } - static simdjson_inline simd8 load(const T values[16]) { - return _mm_loadu_si128(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - __m128i shufmask = _mm_set_epi64x(thintable_epi8[mask2], thintable_epi8[mask1]); - // we increment by 0x08 the second half of the mask - shufmask = - _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); - // this is the version "nearly pruned" - __m128i pruned = _mm_shuffle_epi8(*this, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - __m128i compactmask = - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8)); - __m128i answer = _mm_shuffle_epi8(pruned, compactmask); - _mm_storeu_si128(reinterpret_cast<__m128i *>(output), answer); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - this->chunks[2].store(ptr+sizeof(simd8)*2); - this->chunks[3].store(ptr+sizeof(simd8)*3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - this->chunks[0].compress(uint16_t(mask), output); - this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); - this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); - this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); - return 64 - count_ones(mask); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask() ); - uint64_t r1 = this->chunks[1].to_bitmask() ; - uint64_t r2 = this->chunks[2].to_bitmask() ; - uint64_t r3 = this->chunks[3].to_bitmask() ; - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_SIMD_INPUT_H -/* end file simdjson/westmere/simd.h */ - -namespace simdjson { -namespace westmere { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + 16); - v0.store(dst); - v1.store(dst + 16); - uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; -} - - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H -/* end file simdjson/westmere/stringparsing_defs.h */ -/* end file simdjson/westmere/begin.h */ -/* including generic/amalgamated.h for westmere: #include */ -/* begin file generic/amalgamated.h for westmere */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) -#error generic/dependencies.h must be included before generic/amalgamated.h! -#endif - -/* including generic/base.h for westmere: #include */ -/* begin file generic/base.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { - -struct json_character_block; - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_BASE_H -/* end file generic/base.h for westmere */ -/* including generic/dom_parser_implementation.h for westmere: #include */ -/* begin file generic/dom_parser_implementation.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// Interface a dom parser implementation must fulfill -namespace simdjson { -namespace westmere { -namespace { - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); -simdjson_inline bool is_ascii(const simd8x64& input); - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file generic/dom_parser_implementation.h for westmere */ -/* including generic/json_character_block.h for westmere: #include */ -/* begin file generic/json_character_block.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { - -struct json_character_block { - static simdjson_inline json_character_block classify(const simd::simd8x64& in); - - simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_inline uint64_t op() const noexcept { return _op; } - simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } - - uint64_t _whitespace; - uint64_t _op; -}; - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H -/* end file generic/json_character_block.h for westmere */ -/* end file generic/amalgamated.h for westmere */ -/* including generic/stage1/amalgamated.h for westmere: #include */ -/* begin file generic/stage1/amalgamated.h for westmere */ -// Stuff other things depend on -/* including generic/stage1/base.h for westmere: #include */ -/* begin file generic/stage1/base.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace stage1 { - -class bit_indexer; -template -struct buf_block_reader; -struct json_block; -class json_minifier; -class json_scanner; -struct json_string_block; -class json_string_scanner; -class json_structural_indexer; - -} // namespace stage1 - -namespace utf8_validation { -struct utf8_checker; -} // namespace utf8_validation - -using utf8_validation::utf8_checker; - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H -/* end file generic/stage1/base.h for westmere */ -/* including generic/stage1/buf_block_reader.h for westmere: #include */ -/* begin file generic/stage1/buf_block_reader.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace westmere { -namespace { -namespace stage1 { - -// Walks through a buffer in block-sized increments, loading the last part with spaces -template -struct buf_block_reader { -public: - simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t *full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t *dst) const; - /** - * Like get_remainder, for input followed by SIMDJSON_PADDING readable bytes. - * - * get_remainder writes the block with a memset and a memcpy of the remaining - * length, and the vector loads that read it back cannot be forwarded from - * those narrower stores: they wait until the stores reach the cache, which - * costs tens of cycles on every document. Here the block is put together in - * vector registers, from loads that may read into the padding, and written - * with full-width stores that the loads can forward from. - */ - simdjson_inline size_t get_remainder_padded(uint8_t *dst) const; - simdjson_inline void advance(); -private: - const uint8_t *buf; - const size_t len; - const size_t lenminusstep; - size_t idx; -}; - -template -simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} - -template -simdjson_inline size_t buf_block_reader::block_index() { return idx; } - -template -simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; -} - -template -simdjson_inline const uint8_t *buf_block_reader::full_block() const { - return &buf[idx]; -} - -template -simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { - if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; -} - -// keep_mask[128 - n + i] is 0xFF for i < n and 0 otherwise. -alignas(64) static const uint8_t keep_mask[256] = { - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF -}; - -template -simdjson_inline size_t buf_block_reader::get_remainder_padded(uint8_t *dst) const { - static_assert(STEP_SIZE <= 128, "keep_mask covers blocks of up to 128 bytes"); - static_assert(sizeof(simd::simd8) <= SIMDJSON_PADDING, "a vector load at the last byte must stay in the padding"); - const size_t remaining = len - idx; - if (remaining == 0) { return 0; } - const simd::simd8 spaces = simd::simd8::splat(0x20); - for (size_t off = 0; off < STEP_SIZE; off += sizeof(simd::simd8)) { - simd::simd8 v = spaces; - if (off < remaining) { - // Starts before len, so it ends within the padding. - const simd::simd8 in = simd::simd8::load(buf + idx + off); - const simd::simd8 keep = simd::simd8::load(keep_mask + 128 - remaining + off); - v = spaces ^ ((in ^ spaces) & keep); - } - v.store(dst + off); - } - return remaining; -} - -template -simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for westmere */ -/* including generic/stage1/json_escape_scanner.h for westmere: #include */ -/* begin file generic/stage1/json_escape_scanner.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace stage1 { - -/** - * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). - */ -struct json_escape_scanner { - /** The actual escape characters (the backslashes themselves). */ - uint64_t next_is_escaped = 0ULL; - - struct escaped_and_escape { - /** - * Mask of escaped characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 0100100010100101000 - * n \ \ n \ \ - * ``` - */ - uint64_t escaped; - /** - * Mask of escape characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 1001000101001010001 - * \ \ \ \ \ \ \ - * ``` - */ - uint64_t escape; - }; - - /** - * Get a mask of both escape and escaped characters (the characters following a backslash). - * - * @param potential_escape A mask of the character that can escape others (but could be - * escaped itself). e.g. block.eq('\\') - */ - simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { - -#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT - if (!backslash) { return {next_escaped_without_backslashes(), 0}; } -#endif - - // | | Mask (shows characters instead of 1's) | Depth | Instructions | - // |--------------------------------|----------------------------------------|-------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | - // | | ` even odd even odd odd` | | | - // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) - // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) - // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) - // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () - // (*) this is not needed until the next iteration - uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); - uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); - uint64_t escape = escape_and_terminal_code & backslash; - this->next_is_escaped = escape >> 63; - return {escaped, escape}; - } - -private: - static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; - - simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { - uint64_t escaped = this->next_is_escaped; - this->next_is_escaped = 0; - return escaped; - } - - /** - * Returns a mask of the next escape characters (masking out escaped backslashes), along with - * any non-backslash escape codes. - * - * \n \\n \\\n \\\\n returns: - * \n \ \ \n \ \ - * 11 100 1011 10100 - * - * You are expected to mask out the first bit yourself if the previous block had a trailing - * escape. - * - * & the result with potential_escape to get just the escape characters. - * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. - */ - static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { - // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: - // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be - // inverted (\\\ would be 010 instead of 101). - // - // ``` - // string: | ____\\\\_\\\\_____ | - // maybe_escaped | ODD | \ \ \ \ | - // even-aligned ^^^ ^^^^ odd-aligned - // ``` - // - // Taking that into account, our basic strategy is: - // - // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for - // odd-aligned runs. - // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the - // odd bits in odd-aligned runs. - // 3. & with backslash to clean up any stray bits. - // runs are set to 0, and then XORing with "odd": - // - // | | Mask (shows characters instead of 1's) | Instructions | - // |--------------------------------|----------------------------------------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | - // | | ` even odd even odd odd` | - // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) - // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) - // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) - // - - // Escaped characters are characters following an escape. - uint64_t maybe_escaped = potential_escape << 1; - - // To distinguish odd from even escape sequences, therefore, we turn on any *starting* - // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) - // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. - // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. - // - All other odd bytes are 1, and even bytes are 0. - uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; - uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; - - // Now we flip all odd bytes back with xor. This: - // - Makes odd runs of backslashes go from 0000 to 1010 - // - Makes even runs of backslashes go from 1111 to 1010 - // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) - // - Resets all other bytes to 0 - return even_series_codes_and_odd_bits ^ ODD_BITS; - } -}; - -} // namespace stage1 -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_escape_scanner.h for westmere */ -/* including generic/stage1/json_string_scanner.h for westmere: #include */ -/* begin file generic/stage1/json_string_scanner.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace stage1 { - -struct json_string_block { - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : - _escaped(escaped), _quote(quote), _in_string(in_string) {} - - // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } - // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } - // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } - // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } - - // escaped characters (backslashed--does not include the hex characters after \u) - uint64_t _escaped; - // real quotes (non-escaped ones) - uint64_t _quote; - // string characters (includes start quote but not end quote) - uint64_t _in_string; -}; - -// Scans blocks for string characters, storing the state necessary to do so -class json_string_scanner { -public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); - -private: - // Scans for escape characters - json_escape_scanner escape_scanner{}; - // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). - uint64_t prev_in_string = 0ULL; -}; - -// -// Return a mask of all string characters plus end quotes. -// -// prev_escaped is overflow saying whether the next character is escaped. -// prev_in_string is overflow saying whether we're still in a string. -// -// Backslash sequences outside of quotes will be detected in stage 2. -// -simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = escape_scanner.next(backslash).escaped; - const uint64_t quote = in.eq('"') & ~escaped; - - // - // prefix_xor flips on bits inside the string (and flips off the end quote). - // - // Then we xor with prev_in_string: if we were in a string already, its effect is flipped - // (characters inside strings are outside, and characters outside strings are inside). - // - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - - // - // Check if we're still in a string at the end of the box so the next block will know - // - prev_in_string = uint64_t(static_cast(in_string) >> 63); - - // Use ^ to turn the beginning quote off, and the end quote on. - - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_string_block(escaped, quote, in_string); -} - -simdjson_really_inline error_code json_string_scanner::finish() { - if (prev_in_string) { - return UNCLOSED_STRING; - } - return SUCCESS; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_string_scanner.h for westmere */ -/* including generic/stage1/utf8_lookup4_algorithm.h for westmere: #include */ -/* begin file generic/stage1/utf8_lookup4_algorithm.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace utf8_validation { - -using namespace simd; - - simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { -// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) -// Bit 1 = Too Long (ASCII followed by continuation) -// Bit 2 = Overlong 3-byte -// Bit 4 = Surrogate -// Bit 5 = Overlong 2-byte -// Bit 7 = Two Continuations - constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ - // 11______ 11______ - constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ - constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ - constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ - constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ - constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ - constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ - // 11110100 101_____ - // 11110101 1001____ - // 11110101 101_____ - // 1111011_ 1001____ - // 1111011_ 101_____ - // 11111___ 1001____ - // 11111___ 101_____ - constexpr const uint8_t TOO_LARGE_1000 = 1<<6; - // 11110101 1000____ - // 1111011_ 1000____ - // 11111___ 1000____ - constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ - - const simd8 byte_1_high = prev1.shr<4>().lookup_16( - // 0_______ ________ - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - // 10______ ________ - TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, - // 1100____ ________ - TOO_SHORT | OVERLONG_2, - // 1101____ ________ - TOO_SHORT, - // 1110____ ________ - TOO_SHORT | OVERLONG_3 | SURROGATE, - // 1111____ ________ - TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 - ); - constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( - // ____0000 ________ - CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, - // ____0001 ________ - CARRY | OVERLONG_2, - // ____001_ ________ - CARRY, - CARRY, - - // ____0100 ________ - CARRY | TOO_LARGE, - // ____0101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____011_ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - - // ____1___ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____1101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000 - ); - const simd8 byte_2_high = input.shr<4>().lookup_16( - // ________ 0_______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - - // ________ 1000____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, - // ________ 1001____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, - // ________ 101_____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - - // ________ 11______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT - ); - return (byte_1_high & byte_1_low & byte_2_high); - } - simdjson_inline simd8 check_multibyte_lengths(const simd8 input, - const simd8 prev_input, const simd8 sc) { - simd8 prev2 = input.prev<2>(prev_input); - simd8 prev3 = input.prev<3>(prev_input); - simd8 must23 = must_be_2_3_continuation(prev2, prev3); - simd8 must23_80 = must23 & uint8_t(0x80); - return must23_80 ^ sc; - } - - // - // Return nonzero if there are incomplete multibyte characters at the end of the block: - // e.g. if there is a 4-byte character, but it's 3 bytes from the end. - // - simdjson_inline simd8 is_incomplete(const simd8 input) { - // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): - // ... 1111____ 111_____ 11______ -#if SIMDJSON_IMPLEMENTATION_ICELAKE || (SIMDJSON_IMPLEMENTATION_RVV_VLS && __riscv_v_fixed_vlen >= 512) - static const uint8_t max_array[64] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#else - static const uint8_t max_array[32] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#endif - const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); - return input.gt_bits(max_value); - } - - struct utf8_checker { - // If this is nonzero, there has been a UTF-8 error. - simd8 error; - // The last input we received - simd8 prev_input_block; - // Whether the last input we received was incomplete (used for ASCII fast path) - simd8 prev_incomplete; - - // - // Check whether the current bytes are valid UTF-8. - // - simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { - // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes - // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) - simd8 prev1 = input.prev<1>(prev_input); - simd8 sc = check_special_cases(input, prev1); - this->error |= check_multibyte_lengths(input, prev_input, sc); - } - - // The only problem that can happen at EOF is that a multibyte character is too short - // or a byte value too large in the last bytes: check_special_cases only checks for bytes - // too large in the first of two bytes. - simdjson_inline void check_eof() { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; - } - - simdjson_inline void check_next_input(const simd8x64& input) { - if(simdjson_likely(is_ascii(input))) { - this->error |= this->prev_incomplete; - } else { - // you might think that a for-loop would work, but under Visual Studio, it is not good enough. - static_assert((simd8x64::NUM_CHUNKS == 1) - ||(simd8x64::NUM_CHUNKS == 2) - || (simd8x64::NUM_CHUNKS == 4), - "We support one, two or four chunks per 64-byte block."); - SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 1) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 2) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 4) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - this->check_utf8_bytes(input.get<2>(), input.get<1>()); - this->check_utf8_bytes(input.get<3>(), input.get<2>()); - } - this->prev_incomplete = is_incomplete(input.get::NUM_CHUNKS-1>()); - this->prev_input_block = input.get::NUM_CHUNKS-1>(); - } - } - // do not forget to call check_eof! - simdjson_warn_unused simdjson_inline error_code errors() { - return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; - } - - }; // struct utf8_checker -} // namespace utf8_validation - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H -/* end file generic/stage1/utf8_lookup4_algorithm.h for westmere */ -/* including generic/stage1/json_scanner.h for westmere: #include */ -/* begin file generic/stage1/json_scanner.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace stage1 { - -/** - * A block of scanned json, with information on operators and scalars. - * - * We seek to identify pseudo-structural characters. Anything that is inside - * a string must be omitted (hence & ~_string.string_tail()). - * Otherwise, pseudo-structural characters come in two forms. - * 1. We have the structural characters ([,],{,},:, comma). The - * term 'structural character' is from the JSON RFC. - * 2. We have the 'scalar pseudo-structural characters'. - * Scalars are quotes, and any character except structural characters and white space. - * - * To identify the scalar pseudo-structural characters, we must look at what comes - * before them: it must be a space, a quote or a structural characters. - * Starting with simdjson v0.3, we identify them by - * negation: we identify everything that is followed by a non-quote scalar, - * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. - */ -struct json_block { -public: - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - - /** - * The start of structurals. - * In simdjson prior to v0.3, these were called the pseudo-structural characters. - **/ - simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } - /** All JSON whitespace (i.e. not in a string) */ - simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } - - // Helpers - - /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } - /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } - - // string and escape characters - json_string_block _string; - // whitespace, structural characters ('operators'), scalars - json_character_block _characters; - // whether the previous character was a scalar - uint64_t _follows_potential_nonquote_scalar; -private: - // Potential structurals (i.e. disregarding strings) - - /** - * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". - * They may reside inside a string. - **/ - simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } - /** - * The start of non-operator runs, like 123, true and "abc". - * It main reside inside a string. - **/ - simdjson_inline uint64_t potential_scalar_start() const noexcept { - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space - // then we know that it is irrelevant structurally. - return _characters.scalar() & ~follows_potential_scalar(); - } - /** - * Whether the given character is immediately after a non-operator like 123, true. - * The characters following a quote are not included. - */ - simdjson_inline uint64_t follows_potential_scalar() const noexcept { - // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character - // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a - // white space. - // It is understood that within quoted region, anything at all could be marked (irrelevant). - return _follows_potential_nonquote_scalar; - } -}; - -/** - * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. - * - * The scanner starts by calculating two distinct things: - * - string characters (taking \" into account) - * - structural characters or 'operators' ([]{},:, comma) - * and scalars (runs of non-operators like 123, true and "abc") - * - * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: - * in particular, the operator/scalar bit will find plenty of things that are actually part of - * strings. When we're done, json_block will fuse the two together by masking out tokens that are - * part of a string. - */ -class json_scanner { -public: - json_scanner() = default; - simdjson_inline json_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_warn_unused simdjson_inline error_code finish(); - -private: - // Whether the last character of the previous iteration is part of a scalar token - // (anything except whitespace or a structural character/'operator'). - uint64_t prev_scalar = 0ULL; - json_string_scanner string_scanner{}; -}; - - -// -// Check if the current character immediately follows a matching character. -// -// For example, this checks for quotes with backslashes in front of them: -// -// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); -// -simdjson_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; -} - -simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { - json_string_block strings = string_scanner.next(in); - // identifies the white-space and the structural characters - json_character_block characters = json_character_block::classify(in); - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). - // - // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) - // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential - // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we - // may need to add an extra check when parsing strings. - // - // Performance: there are many ways to skin this cat. - const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); - uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_block( - strings,// strings is a function-local object so either it moves or the copy is elided. - characters, - follows_nonquote_scalar - ); -} - -simdjson_warn_unused simdjson_inline error_code json_scanner::finish() { - return string_scanner.finish(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H -/* end file generic/stage1/json_scanner.h for westmere */ - -// All other declarations -/* including generic/stage1/find_next_document_index.h for westmere: #include */ -/* begin file generic/stage1/find_next_document_index.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace stage1 { - -/** - * This algorithm is used to quickly identify the last structural position that - * makes up a complete document. - * - * It does this by going backwards and finding the last *document boundary* (a - * place where one value follows another without a comma between them). If the - * last document (the characters after the boundary) has an equal number of - * start and end brackets, it is considered complete. - * - * Simply put, we iterate over the structural characters, starting from - * the end. We consider that we found the end of a JSON document when the - * first element of the pair is NOT one of these characters: '{' '[' ':' ',' - * and when the second element is NOT one of these characters: '}' ']' ':' ','. - * - * This simple comparison works most of the time, but it does not cover cases - * where the batch's structural indexes contain a perfect amount of documents. - * In such a case, we do not have access to the structural index which follows - * the last document, therefore, we do not have access to the second element in - * the pair, and that means we cannot identify the last document. To fix this - * issue, we keep a count of the open and closed curly/square braces we found - * while searching for the pair. When we find a pair AND the count of open and - * closed curly/square braces is the same, we know that we just passed a - * complete document, therefore the last json buffer location is the end of the - * batch. - */ -simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { - const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - const uint8_t e = parser.buf[len - 1]; - return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && - e != ' ' && e != '\t' && e != '\n' && e != '\r'; -} - -/** - * The structural index past the last one of a partial batch is the end of this - * window, not a token of it: the byte there is the first byte of the next - * window. A finished value must not be rejected because that byte is ',', ':', - * ']' or '}'. A last document that itself ends with '{', '[', ',' or ':' is - * invalid JSON, yet its brackets may balance: walking it, the On-Demand API - * could read the next batch as a value. We leave such a document to the next - * batch, so that the batch ends between two documents. - */ -simdjson_inline bool ends_with_unfinished_document(dom_parser_implementation &parser) { - const uint8_t last = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - return last == '{' || last == '[' || last == ',' || last == ':'; -} - -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { - // Variant: do not count separately, just figure out depth - if(parser.n_structural_indexes == 0) { return 0; } - auto arr_cnt = 0; - auto obj_cnt = 0; - for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { - auto idxb = parser.structural_indexes[i]; - switch (parser.buf[idxb]) { - case ':': - case ',': - continue; - case '}': - obj_cnt--; - continue; - case ']': - arr_cnt--; - continue; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - auto idxa = parser.structural_indexes[i - 1]; - switch (parser.buf[idxa]) { - case '{': - case '[': - case ':': - case ',': - continue; - } - // Last document is complete, so the next document will appear after! - if (!arr_cnt && !obj_cnt) { - return defer_last ? i : parser.n_structural_indexes; - } - // Last document is incomplete; mark the document at i + 1 as the next one - return i; - } - // If we made it to the end, we want to finish counting to see if we have a full document. - switch (parser.buf[parser.structural_indexes[0]]) { - case '}': - obj_cnt--; - break; - case ']': - arr_cnt--; - break; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - if (!arr_cnt && !obj_cnt) { - // We have a complete document. - return defer_last ? 0 : parser.n_structural_indexes; - } - return 0; -} - -/** - * Sentinel value returned to indicate a document started but didn't fit - * (CAPACITY error), as opposed to 0 which means no document content found - * (EMPTY). - */ -constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; - -/** - * For RFC 7464 JSON text sequences, filter RS from structural indexes and - * find batch boundaries. - * - * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. - * RS bytes appear in structural_indexes as they are classified as scalars. - * This function: - * 1. Scans structural_indexes to find and count RS positions - * 2. Filters RS out of structural_indexes in-place - * 3. Determines batch boundaries based on RS positions - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @param scan_len Offset past which no value start may be derived. Equals len - * unless stage 1 dropped a trailing unclosed string, whose bytes it - * never validated. - * @return The number of structural indexes to keep (after RS filtering), - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t find_next_document_index_json_sequence( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start, - size_t scan_len) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Phase 1: Scan structural_indexes to find RS positions and handle them. - // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS - // is already in structural_indexes (it's an operator). For scalars like numbers, - // the digit following RS is NOT in structural_indexes because the scanner sees - // RS as a scalar, making the digit a scalar continuation, not a start. - // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the - // actual value start position. - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_rs_pos = 0; - uint32_t rs_count = 0; - - for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { - const uint32_t pos = parser.structural_indexes[read_idx]; - if (parser.buf[pos] == 0x1E) { - // This is an RS character - find the actual JSON value start. - last_rs_pos = pos; - rs_count++; - // Skip past this RS and any whitespace *and any additional RSes* - // to locate the real value. Consecutive RSes are degenerate - // "empty records" per RFC 7464; we collapse them here. They do - // not always appear as separate entries in structural_indexes - // because the scanner groups runs of adjacent non-whitespace - // scalar bytes (including RS) into a single scalar start. - uint32_t value_start = pos + 1; - while (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { - value_start++; - } else if (c == 0x1E) { - // Collapsed empty record. Still count it so rs_count reflects - // the true number of record markers and last_rs_pos tracks - // the final one. - last_rs_pos = value_start; - rs_count++; - value_start++; - } else { - break; - } - } - // If the scanner emitted additional structurals inside the - // whitespace+RS run we just walked over (i.e., isolated RSes - // separated by whitespace), skip past them so we do not - // double-count or double-emit. - while (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] < value_start) { - read_idx++; - } - // Check if the value start is an operator (always present in - // scanner structural_indexes) or a scalar-like start (which may - // be missing from structural_indexes and must be added here). - // Note: '"' is NOT always in structural_indexes. The scanner - // classifies '"' as a scalar character and emits it as a - // structural only when it is a *scalar start* (preceded by - // whitespace or an operator). When '"' immediately follows an - // RS (which the scanner also classifies as scalar), it is - // treated as a scalar continuation and not emitted - so we - // must add it here just like any other scalar value. - if (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - const bool is_operator = - (c == '{' || c == '}' || c == '[' || c == ']' || - c == ':' || c == ','); - // If the next scanner structural is exactly at value_start, - // the scanner already emitted it (it followed whitespace) and - // we must not add a duplicate - a subsequent iteration will - // copy it into write_idx. - const bool already_emitted = - (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] == value_start); - if (!is_operator && !already_emitted) { - // Scalar value (number/true/false/null/string) - add its - // position since scanner missed it. - parser.structural_indexes[write_idx++] = value_start; - } - } - } else { - // Not RS, copy to output - parser.structural_indexes[write_idx++] = pos; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { - // Only RS markers here: the last one opens a record continuing past the - // window, so that is where the next batch resumes. - if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } - return 0; - } - if (rs_count == 0) { - // No RS found; for final batch, try generic boundary detection - return is_final ? find_next_document_index(parser) : 0; - } - - // Phase 2: Determine batch boundaries based on RS positions - - if (is_final) { - // Final batch: all documents are complete (last one ends at EOF). - // In json_sequence mode, RS markers define document boundaries, so all - // remaining structurals form complete documents. Return them all directly. - // (Calling find_next_document_index() would fail for scalar documents.) - return parser.n_structural_indexes; - } - - // Partial batch: need to find complete documents only. - // A document starting at an RS is complete if there is another RS after it. - next_batch_start = last_rs_pos; - - if (rs_count < 2) { - // Only one RS, so we have at most one document that may be incomplete. - // We cannot confirm it is complete without another RS. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least 2 RS markers. The last complete document ends before last_rs_pos. - - // Find the structural index cutoff: keep only structurals < last_rs_pos. - // Since we already filtered RS, all remaining structurals are valid. - // We iterate backward to find the last structural before last_rs_pos. - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_rs_pos) { - keep_count = i; - break; - } - } - - // No structurals before the last RS - no complete documents - if (keep_count == 0) { return 0; } - - // All documents before the last RS are complete by definition (the next RS - // confirms their end). No need to call find_next_document_index() which - // would fail for scalar documents like `1` or `"hello"`. - return keep_count; -} - -/** - * Filter comma-delimited documents by removing root-level commas from - * structural indexes. - * - * For comma-delimited format like `{...},{...},{...}`, we need to remove - * the commas that separate documents (depth 0) while preserving commas - * inside arrays and objects (depth > 0). - * - * After filtering, the structural indexes look like whitespace-delimited - * documents, so find_next_document_index() works unchanged. - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @return The number of structural indexes to keep, - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t filter_comma_delimited( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Track depth to identify root-level commas (depth 0) - int depth = 0; - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_root_comma_pos = 0; - uint32_t root_comma_count = 0; - - for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { - uint32_t idx = parser.structural_indexes[i]; - uint8_t c = parser.buf[idx]; - - switch (c) { - case '{': case '[': - depth++; - parser.structural_indexes[write_idx++] = idx; - break; - case '}': case ']': - depth--; - parser.structural_indexes[write_idx++] = idx; - break; - case ',': - if (depth == 0) { - // Root-level comma = document boundary, skip it - last_root_comma_pos = idx; - root_comma_count++; - continue; - } - parser.structural_indexes[write_idx++] = idx; - break; - default: - // Colons, scalars, etc. - parser.structural_indexes[write_idx++] = idx; - break; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { return 0; } - - if (is_final) { - // Final batch: use standard boundary detection on filtered indexes - return find_next_document_index(parser); - } - - // Partial batch: need to find complete documents only. - // A document ending with a root comma is complete. - if (root_comma_count == 0) { - // No root commas found; we cannot confirm any document is complete. - // The whole batch might be one incomplete document. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least one root comma. Documents before the last comma are complete. - next_batch_start = last_root_comma_pos + 1; - - // Find the structural index cutoff: keep only structurals < last_root_comma_pos - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_root_comma_pos) { - keep_count = i; - break; - } - } - - if (keep_count == 0) { return 0; } - - // Use standard boundary detection on the complete portion - parser.n_structural_indexes = keep_count; - return find_next_document_index(parser); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H -/* end file generic/stage1/find_next_document_index.h for westmere */ -/* including generic/stage1/json_minifier.h for westmere: #include */ -/* begin file generic/stage1/json_minifier.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace westmere { -namespace { -namespace stage1 { - -class json_minifier { -public: - template - static error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept; - -private: - simdjson_inline json_minifier(uint8_t *_dst) - : dst{_dst} - {} - template - simdjson_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_warn_unused simdjson_inline error_code finish(uint8_t *dst_start, size_t &dst_len); - json_scanner scanner{}; - uint8_t *dst; -}; - -simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { - uint64_t mask = block.whitespace(); - dst += in.compress(mask, dst); -} - -simdjson_warn_unused simdjson_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { - error_code error = scanner.finish(); - if (error) { dst_len = 0; return error; } - dst_len = dst - dst_start; - return SUCCESS; -} - -template<> -simdjson_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - simd::simd8x64 in_2(block_buf+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1); - this->next(in_2, block_2); - reader.advance(); -} - -template<> -simdjson_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - json_block block_1 = scanner.next(in_1); - this->next(block_buf, block_1); - reader.advance(); -} - -template -error_code json_minifier::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept { - buf_block_reader reader(buf, len); - json_minifier minifier(dst); - - // Index the first n-1 blocks - while (reader.has_full_block()) { - minifier.step(reader.full_block(), reader); - } - - // Index the last (remainder) block, padded with spaces - uint8_t block[STEP_SIZE]; - size_t remaining_bytes = reader.get_remainder(block); - if (remaining_bytes > 0) { - // We do not want to write directly to the output stream. Rather, we write - // to a local buffer (for safety). - uint8_t out_block[STEP_SIZE]; - uint8_t * const guarded_dst{minifier.dst}; - minifier.dst = out_block; - minifier.step(block, reader); - size_t to_write = minifier.dst - out_block; - // In some cases, we could be enticed to consider the padded spaces - // as part of the string. This is fine as long as we do not write more - // than we consumed. - if(to_write > remaining_bytes) { to_write = remaining_bytes; } - memcpy(guarded_dst, out_block, to_write); - minifier.dst = guarded_dst + to_write; - } - return minifier.finish(dst, dst_len); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H -/* end file generic/stage1/json_minifier.h for westmere */ -/* including generic/stage1/json_structural_indexer.h for westmere: #include */ -/* begin file generic/stage1/json_structural_indexer.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace westmere { -namespace { -namespace stage1 { - -class bit_indexer { -public: - uint32_t *tail; - - simdjson_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} - -#if SIMDJSON_PREFER_REVERSE_BITS - /** - * ARM lacks a fast trailing zero instruction, but it has a fast - * bit reversal instruction and a fast leading zero instruction. - * Thus it may be profitable to reverse the bits (once) and then - * to rely on a sequence of instructions that call the leading - * zero instruction. - * - * Performance notes: - * The chosen routine is not optimal in terms of data dependency - * since zero_leading_bit might require two instructions. However, - * it tends to minimize the total number of instructions which is - * beneficial. - */ - simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { - int lz = leading_zeroes(rev_bits); - this->tail[i] = static_cast(idx) + lz; - rev_bits = zero_leading_bit(rev_bits, lz); - } -#else - /** - * Under recent x64 systems, we often have both a fast trailing zero - * instruction and a fast 'clear-lower-bit' instruction so the following - * algorithm can be competitive. - */ - - simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { - this->tail[i] = idx + trailing_zeroes(bits); - bits = clear_lowest_bit(bits); - } -#endif // SIMDJSON_PREFER_REVERSE_BITS - - template - simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { - write_index(idx, bits, START); - SIMDJSON_IF_CONSTEXPR (N > 1) { - write_indexes<(N-1>0?START+1:START), (N-1>=0?N-1:1)>(idx, bits); - } - return START+N; - } - - template - simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { - write_indexes(idx, bits); - SIMDJSON_IF_CONSTEXPR ((START+STEP) < END) { - if (simdjson_unlikely((START+STEP) < cnt)) { - write_indexes_stepped<(START+STEP(idx, bits, cnt); - } - } - return ((END-START) % STEP) == 0 ? END : (END-START) - ((END-START) % STEP) + STEP; - } - - // flatten out values in 'bits' assuming that they are are to have values of idx - // plus their position in the bitvector, and store these indexes at - // base_ptr[base] incrementing base as we go - // will potentially store extra values beyond end of valid bits, so base_ptr - // needs to be large enough to handle this - // - // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it - // will provide its own version of the code. -#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - simdjson_inline void write(uint32_t idx, uint64_t bits); -#else - simdjson_inline void write(uint32_t idx, uint64_t bits) { - // In some instances, the next branch is expensive because it is mispredicted. - // Unfortunately, in other cases, - // it helps tremendously. - if (bits == 0) - return; - - int cnt = static_cast(count_ones(bits)); - -#if SIMDJSON_PREFER_REVERSE_BITS - bits = reverse_bits(bits); -#endif -#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP - static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; -#else - static constexpr const int STEP = 4; -#endif - static constexpr const int STEP_UNTIL = 24; - - write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); - SIMDJSON_IF_CONSTEXPR (STEP_UNTIL < 64) { - if (simdjson_unlikely(STEP_UNTIL < cnt)) { - for (int i=STEP_UNTIL; itail += cnt; - } -#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -}; - -class json_structural_indexer { -public: - /** - * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. - * - * @param partial Setting the partial parameter to true allows the find_structural_bits to - * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If - * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. - */ - template - static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept; - -private: - simdjson_inline json_structural_indexer(uint32_t *structural_indexes); - template - simdjson_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_warn_unused simdjson_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); - - json_scanner scanner{}; - utf8_checker checker{}; - bit_indexer indexer; - uint64_t prev_structurals = 0; - uint64_t unescaped_chars_error = 0; -}; - -simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} - -// Skip the last character if it is partial -simdjson_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { - if (simdjson_unlikely(len < 3)) { - switch (len) { - case 2: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left - return len; - case 1: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - return len; - case 0: - return len; - } - } - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left - return len; -} - -// -// PERF NOTES: -// We pipe 2 inputs through these stages: -// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load -// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. -// 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. -// The output of step 1 depends entirely on this information. These functions don't quite use -// up enough CPU: the second half of the functions is highly serial, only using 1 execution core -// at a time. The second input's scans has some dependency on the first ones finishing it, but -// they can make a lot of progress before they need that information. -// 3. Step 1 does not use enough capacity, so we run some extra stuff while we're waiting for that -// to finish: utf-8 checks and generating the output from the last iteration. -// -// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all -// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough -// workout. -// -template -error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept { - if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } - // We guard the rest of the code so that we can assume that len > 0 throughout. - if (len == 0) { return EMPTY; } - if (is_streaming(partial)) { - len = trim_partial_utf8(buf, len); - // If you end up with an empty window after trimming - // the partial UTF-8 bytes, then chances are good that you - // have an UTF-8 formatting error. - if(len == 0) { return UTF8_ERROR; } - } - buf_block_reader reader(buf, len); - json_structural_indexer indexer(parser.structural_indexes.get()); - - // Read all but the last block - while (reader.has_full_block()) { - indexer.step(reader.full_block(), reader); - } - // Take care of the last block (will always be there unless file is empty which is - // not supposed to happen.) - uint8_t block[STEP_SIZE]; - const size_t remaining = parser._unpadded ? reader.get_remainder(block) : reader.get_remainder_padded(block); - if (simdjson_unlikely(remaining == 0)) { return UNEXPECTED_ERROR; } - indexer.step(block, reader); - return indexer.finish(parser, reader.block_index(), len, partial); -} - -template<> -simdjson_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block); - simd::simd8x64 in_2(block+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1, reader.block_index()); - this->next(in_2, block_2, reader.block_index()+64); - reader.advance(); -} - -template<> -simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block); - json_block block_1 = scanner.next(in_1); - this->next(in_1, block_1, reader.block_index()); - reader.advance(); -} - -simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { - uint64_t unescaped = in.lteq(0x1F); -#if SIMDJSON_UTF8VALIDATION - checker.check_next_input(in); -#endif - indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser - prev_structurals = block.structural_start(); - unescaped_chars_error |= block.non_quote_inside_string(unescaped); -} - -// Drops the unclosed string that ends a json_sequence batch and leaves its -// start in structural_indexes[n_structural_indexes]. The last index is its -// opening quote, unless the quote directly follows an RS: the scanner reads -// RS as a scalar byte, so it indexes the RS and not the quote. Keep the RS. -simdjson_inline void drop_json_sequence_unclosed_string(dom_parser_implementation &parser, size_t len) noexcept { - uint32_t quote = parser.structural_indexes[parser.n_structural_indexes - 1]; - if (parser.buf[quote] == '"') { - parser.n_structural_indexes--; - return; - } - while (quote < len && parser.buf[quote] != '"') { quote++; } - parser.structural_indexes[parser.n_structural_indexes] = quote; -} - -simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial) { - // Write out the final iteration's structurals - indexer.write(uint32_t(idx-64), prev_structurals); - error_code error = scanner.finish(); - // We deliberately break down the next expression so that it is - // human readable. - const bool should_we_exit = is_streaming(partial) ? - ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING - : (error != SUCCESS); // if partial is false, we must have SUCCESS - const bool have_unclosed_string = (error == UNCLOSED_STRING); - parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); - /*** - * The On-Demand API requires special padding. - * - * This is related to https://github.com/simdjson/simdjson/issues/906 - * Basically, we want to make sure that if the parsing continues beyond the last (valid) - * structural character, it quickly stops. - * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. - * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing - * continues, then it must be [,] or }. - * Suppose it is ] or }. We backtrack to the first character, what could it be that would - * not trigger an error? It could be ] or } but no, because you can't start a document that way. - * It can't be a comma, a colon or any simple value. So the only way we could continue is - * if the repeated character is [. But if so, the document must start with [. But if the document - * starts with [, it should end with ]. If we enforce that rule, then we would get - * ][[ which is invalid. - * - * This is illustrated with the test array_iterate_unclosed_error() on the following input: - * R"({ "a": [,,)" - **/ - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final - parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); - parser.structural_indexes[parser.n_structural_indexes + 2] = 0; - parser.next_structural_index = 0; - - // Bail out only once the count and sentinels above are set: - // document_stream::truncated_bytes() reads them even on error. - if (simdjson_unlikely(should_we_exit)) { return error; } - if (unescaped_chars_error) { return UNESCAPED_CHARS; } - - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - return EMPTY; - } - if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { - return UNEXPECTED_ERROR; - } - if (partial == stage1_mode::streaming_partial) { - // If we have an unclosed string, then the last structural - // will be the quote and we want to make sure to omit it. - if(have_unclosed_string) { - parser.n_structural_indexes--; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - // We truncate the input to the end of the last complete document (or zero). - const bool partial_scalar = !have_unclosed_string && ends_with_partial_scalar(parser, len); - const bool unfinished = ends_with_unfinished_document(parser); - auto new_structural_indexes = find_next_document_index(parser, partial_scalar || unfinished); - if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { - // The batch holds a single document, whose brackets balance but which - // cannot be valid JSON: it is not too big, it is invalid. - if (unfinished && find_next_document_index(parser, partial_scalar) != 0) { return TAPE_ERROR; } - if(parser.structural_indexes[0] == 0) { - // If the buffer is partial and we started at index 0 but the document is - // incomplete, it's too big to parse. - return CAPACITY; - } else { - // It is possible that the document could be parsed, we just had a lot - // of white space. - parser.n_structural_indexes = 0; - return EMPTY; - } - } - parser.n_structural_indexes = new_structural_indexes; - } else if (partial == stage1_mode::streaming_final) { - if(have_unclosed_string) { parser.n_structural_indexes--; } - // We truncate the input to the end of the last complete document (or zero). - // Because partial == stage1_mode::streaming_final, it means that we may - // silently ignore trailing garbage. Though it sounds bad, we do it - // deliberately because many people who have streams of JSON documents - // will truncate them for processing. E.g., imagine that you are uncompressing - // the data from a size file or receiving it in chunks from the network. You - // may not know where exactly the last document will be. Meanwhile the - // document_stream instances allow people to know the JSON documents they are - // parsing (see the iterator.source() method). - parser.n_structural_indexes = find_next_document_index(parser); - // We store the initial n_structural_indexes so that the client can see - // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, - // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, - // otherwise, it will copy some prior index. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - // This next line is critical, do not change it unless you understand what you are - // doing. - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - // We tolerate an unclosed string at the very end of the stream. Indeed, users - // often load their data in bulk without being careful and they want us to ignore - // the trailing garbage. - return EMPTY; - } - } else if (partial == stage1_mode::json_sequence_partial) { - // RFC 7464: use RS positions for batch boundaries - // A discarded unclosed string also caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - // The next batch starts at an RS, which the On-Demand API never reads as a - // string, a container or a separator. But an invalid last document may make - // it skip that RS as a value: repeat it, so that the index past it is not - // the next record, which can be a string that does not close in this batch. - parser.structural_indexes[parser.n_structural_indexes + 1] = next_batch_start; - } else if (partial == stage1_mode::json_sequence_final) { - // RFC 7464: final batch, last document extends to EOF - // As above: a discarded unclosed string caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); - // The filter compacted structural_indexes in place and restored the EOF - // sentinel past the compacted end, so the copy below is either the start - // of a truncated document or len, as in streaming_final. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } else if (partial == stage1_mode::comma_delimited_partial) { - // Comma-delimited: filter root-level commas, use comma positions for batch boundaries - if(have_unclosed_string) { - parser.n_structural_indexes--; - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - } else if (partial == stage1_mode::comma_delimited_final) { - // Comma-delimited: final batch, last document extends to EOF - if(have_unclosed_string) { parser.n_structural_indexes--; } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } - checker.check_eof(); - return checker.errors(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. -#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H -/* end file generic/stage1/json_structural_indexer.h for westmere */ -/* including generic/stage1/utf8_validator.h for westmere: #include */ -/* begin file generic/stage1/utf8_validator.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace stage1 { - -/** - * Validates that the string is actual UTF-8. - */ -template -bool generic_validate_utf8(const uint8_t * input, size_t length) { - checker c{}; - buf_block_reader<64> reader(input, length); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - c.check_next_input(in); - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - c.check_next_input(in); - reader.advance(); - c.check_eof(); - return c.errors() == error_code::SUCCESS; -} - -bool generic_validate_utf8(const char * input, size_t length) { - return generic_validate_utf8(reinterpret_cast(input),length); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H -/* end file generic/stage1/utf8_validator.h for westmere */ -/* end file generic/stage1/amalgamated.h for westmere */ -/* including generic/stage2/amalgamated.h for westmere: #include */ -/* begin file generic/stage2/amalgamated.h for westmere */ -// Stuff other things depend on -/* including generic/stage2/base.h for westmere: #include */ -/* begin file generic/stage2/base.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace stage2 { - -class json_iterator; -struct tape_builder; -struct tape_writer; - -} // namespace stage2 -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H -/* end file generic/stage2/base.h for westmere */ -/* including generic/stage2/tape_writer.h for westmere: #include */ -/* begin file generic/stage2/tape_writer.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace westmere { -namespace { -namespace stage2 { - -struct tape_writer { - /** The next place to write to tape */ - uint64_t *next_tape_loc; - - /** Write a signed 64-bit value to tape. */ - simdjson_inline void append_s64(int64_t value) noexcept; - - /** Write an unsigned 64-bit value to tape. */ - simdjson_inline void append_u64(uint64_t value) noexcept; - - /** Write a double value to tape. */ - simdjson_inline void append_double(double value) noexcept; - - /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ - simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; - - /** - * Append a tape entry (an 8-bit type,and 56 bits worth of value). - */ - simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; - - /** - * Skip the current tape entry without writing. - * - * Used to skip the start of the container, since we'll come back later to fill it in when the - * container ends. - */ - simdjson_inline void skip() noexcept; - - /** - * Skip the number of tape entries necessary to write a large u64 or i64. - */ - simdjson_inline void skip_large_integer() noexcept; - - /** - * Skip the number of tape entries necessary to write a double. - */ - simdjson_inline void skip_double() noexcept; - - /** - * Write a value to a known location on tape. - * - * Used to go back and write out the start of a container after the container ends. - */ - simdjson_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; - -private: - /** - * Append both the tape entry, and a supplementary value following it. Used for types that need - * all 64 bits, such as double and uint64_t. - */ - template - simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; -}; // struct tape_writer - -simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { - append2(0, value, internal::tape_type::INT64); -} - -simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { - append(0, internal::tape_type::UINT64); - *next_tape_loc = value; - next_tape_loc++; -} - -/** Write a double value to tape. */ -simdjson_inline void tape_writer::append_double(double value) noexcept { - append2(0, value, internal::tape_type::DOUBLE); -} - -simdjson_inline void tape_writer::skip() noexcept { - next_tape_loc++; -} - -simdjson_inline void tape_writer::skip_large_integer() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::skip_double() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { - *next_tape_loc = val | ((uint64_t(char(t))) << 56); - next_tape_loc++; -} - -template -simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { - append(val, t); - static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); - memcpy(next_tape_loc, &val2, sizeof(val2)); - next_tape_loc++; -} - -simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { - tape_loc = val | ((uint64_t(char(t))) << 56); -} - -simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { - // Write to string buffer: [4-byte LE length][digits][null] - uint32_t str_len = uint32_t(len); - memcpy(string_buf, &str_len, sizeof(uint32_t)); - memcpy(string_buf + sizeof(uint32_t), src, len); - string_buf[sizeof(uint32_t) + len] = 0; - // Tape entry: offset into string buffer - // The caller must set the offset relative to doc.string_buf base - append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches - string_buf += sizeof(uint32_t) + len + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H -/* end file generic/stage2/tape_writer.h for westmere */ -/* including generic/stage2/logger.h for westmere: #include */ -/* begin file generic/stage2/logger.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - - -// This is for an internal-only stage 2 specific logger. -// Set LOG_ENABLED = true to log what stage 2 is doing! -namespace simdjson { -namespace westmere { -namespace { -namespace logger { - - static constexpr const char * DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; - -#if SIMDJSON_VERBOSE_LOGGING - static constexpr const bool LOG_ENABLED = true; -#else - static constexpr const bool LOG_ENABLED = false; -#endif - static constexpr const int LOG_EVENT_LEN = 20; - static constexpr const int LOG_BUFFER_LEN = 30; - static constexpr const int LOG_SMALL_BUFFER_LEN = 10; - static constexpr const int LOG_INDEX_LEN = 5; - - static int log_depth; // Not threadsafe. Log only. - - // Helper to turn unprintable or newline characters into spaces - static simdjson_inline char printable_char(char c) { - if (c >= 0x20) { - return c; - } else { - return ' '; - } - } - - // Print the header and set up log_start - static simdjson_inline void log_start() { - if (LOG_ENABLED) { - log_depth = 0; - printf("\n"); - printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); - printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN+2, DASHES, LOG_BUFFER_LEN+2, DASHES, LOG_SMALL_BUFFER_LEN+2, DASHES, 5+2, DASHES); - } - } - - simdjson_unused static simdjson_inline void log_string(const char *message) { - if (LOG_ENABLED) { - printf("%s\n", message); - } - } - - // Logs a single line from the stage 2 DOM parser - template - static simdjson_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { - if (LOG_ENABLED) { - printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); - auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; - auto next_index = structurals.next_structural; - auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); - auto next = &structurals.buf[*next_index]; - { - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i=0;i */ -/* begin file generic/stage2/json_iterator.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace stage2 { - -class json_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - uint32_t depth{0}; - - /** - * Walk the JSON document. - * - * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as - * the first parameter; some callbacks have other parameters as well: - * - * - visit_document_start() - at the beginning. - * - visit_document_end() - at the end (if things were successful). - * - * - visit_array_start() - at the start `[` of a non-empty array. - * - visit_array_end() - at the end `]` of a non-empty array. - * - visit_empty_array() - when an empty array is encountered. - * - * - visit_object_end() - at the start `]` of a non-empty object. - * - visit_object_start() - at the end `]` of a non-empty object. - * - visit_empty_object() - when an empty object is encountered. - * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is - * guaranteed to point at the first quote of the string (`"key"`). - * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. - * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. - * - * - increment_count(iter) - each time a value is found in an array or object. - */ - template - simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; - - /** - * Create an iterator capable of walking a JSON document. - * - * The document must have already passed through stage 1. - */ - simdjson_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); - - /** - * Look at the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *peek(const uint32_t *batch_end = nullptr) const noexcept; - /** - * Advance to the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *advance(const uint32_t *batch_end = nullptr) noexcept; - /** - * Get the remaining length of the document, from the start of the current token. - */ - simdjson_inline size_t remaining_len() const noexcept; - /** - * Check if we are at the end of the document. - * - * If this is true, there are no more tokens. - */ - simdjson_inline bool at_eof() const noexcept; - /** - * Check if we are at the beginning of the document. - */ - simdjson_inline bool at_beginning() const noexcept; - simdjson_inline uint8_t last_structural() const noexcept; - - /** - * Log that a value has been found. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_value(const char *type) const noexcept; - /** - * Log the start of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_start_value(const char *type) const noexcept; - /** - * Log the end of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_end_value(const char *type) const noexcept; - /** - * Log an error. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_error(const char *error) const noexcept; - - template - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; - template - simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; -}; - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { - logger::log_start(); - - // - // Start the document - // - if (at_eof()) { return EMPTY; } - // Only used when BOUNDED (see advance()); a local, so that it can stay in a - // register, and vanishes from the other walks. - const uint32_t *const batch_end = &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - log_start_value("document"); - SIMDJSON_TRY( visitor.visit_document_start(*this) ); - - // - // Read first value - // - { - auto value = advance(batch_end); - - // Make sure the outer object or array is closed before continuing; otherwise, there are ways we - // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 - if (!STREAMING) { - switch (*value) { - case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; - case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; - } - } - - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; - } - } - goto document_end; - -// -// Object parser states -// -object_begin: - log_start_value("object"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = false; - SIMDJSON_TRY( visitor.visit_object_start(*this) ); - - { - auto key = advance(batch_end); - if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.increment_count(*this) ); - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - -object_field: - if (simdjson_unlikely( (*advance(batch_end) != ':') )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -object_continue: - switch (*advance(batch_end)) { - case ',': - SIMDJSON_TRY( visitor.increment_count(*this) ); - { - auto key = advance(batch_end); - if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - goto object_field; - case '}': log_end_value("object"); SIMDJSON_TRY( visitor.visit_object_end(*this) ); goto scope_end; - default: log_error("No comma between object fields"); return TAPE_ERROR; - } - -scope_end: - depth--; - if (depth == 0) { goto document_end; } - if (dom_parser.is_array[depth]) { goto array_continue; } - goto object_continue; - -// -// Array parser states -// -array_begin: - log_start_value("array"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = true; - SIMDJSON_TRY( visitor.visit_array_start(*this) ); - SIMDJSON_TRY( visitor.increment_count(*this) ); - -array_value: - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -array_continue: - switch (*advance(batch_end)) { - case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; - case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; - default: log_error("Missing comma between array values"); return TAPE_ERROR; - } - -document_end: - log_end_value("document"); - SIMDJSON_TRY( visitor.visit_document_end(*this) ); - - dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); - - // If we didn't make it to the end, it's an error - if ( !STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes ) { - log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); - return TAPE_ERROR; - } - - return SUCCESS; - -} // walk_document() - -simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { -} - -// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires -// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). -// -// When BOUNDED (see dom_parser_implementation::_bounded_batch), a document may -// not run past batch_end, one past the last structural index of the batch: -// past it, we return the same dummy, and walk_document fails cleanly. -template -simdjson_inline const uint8_t *json_iterator::peek(const uint32_t *batch_end) const noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -template -simdjson_inline const uint8_t *json_iterator::advance(const uint32_t *batch_end) noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural++); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -simdjson_inline size_t json_iterator::remaining_len() const noexcept { - return dom_parser.len - *(next_structural-1); -} - -simdjson_inline bool json_iterator::at_eof() const noexcept { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; -} -simdjson_inline bool json_iterator::at_beginning() const noexcept { - return next_structural == dom_parser.structural_indexes.get(); -} -simdjson_inline uint8_t json_iterator::last_structural() const noexcept { - return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; -} - -simdjson_inline void json_iterator::log_value(const char *type) const noexcept { - logger::log_line(*this, "", type, ""); -} - -simdjson_inline void json_iterator::log_start_value(const char *type) const noexcept { - logger::log_line(*this, "+", type, ""); - if (logger::LOG_ENABLED) { logger::log_depth++; } -} - -simdjson_inline void json_iterator::log_end_value(const char *type) const noexcept { - if (logger::LOG_ENABLED) { logger::log_depth--; } - logger::log_line(*this, "-", type, ""); -} - -simdjson_inline void json_iterator::log_error(const char *error) const noexcept { - logger::log_line(*this, "", "ERROR", error); -} - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { - switch (*value) { - case '"': return visitor.visit_root_string(*this, value); - case 't': return visitor.visit_root_true_atom(*this, value); - case 'f': return visitor.visit_root_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_root_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_root_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_root_inf_atom(*this, value); -#else - case 'n': return visitor.visit_root_null_atom(*this, value); -#endif - case '-': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return visitor.visit_root_number(*this, value); - default: - log_error("Document starts with a non-value character"); - return TAPE_ERROR; - } -} -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { - // Use the fact that most scalars are going to be either strings or numbers. - if(*value == '"') { - return visitor.visit_string(*this, value); - } else if (((*value - '0') < 10) || (*value == '-')) { - return visitor.visit_number(*this, value); - } - // true, false, null are uncommon. - switch (*value) { - case 't': return visitor.visit_true_atom(*this, value); - case 'f': return visitor.visit_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_inf_atom(*this, value); -#else - case 'n': return visitor.visit_null_atom(*this, value); -#endif - default: - log_error("Non-value found when value was expected!"); - return TAPE_ERROR; - } -} - -} // namespace stage2 -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H -/* end file generic/stage2/json_iterator.h for westmere */ -/* including generic/stage2/stringparsing.h for westmere: #include */ -/* begin file generic/stage2/stringparsing.h for westmere */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace westmere { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - // Decode adjacent Unicode escapes without returning to the - // quote-and-backslash scanner between code points. - do { - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } while (src[0] == '\\' && src[1] == 'u'); - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -/** - * Bounds-safe variant of parse_string for input buffers that are NOT padded to - * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input - * byte (buf + len). It behaves exactly like parse_string while we are at least - * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays - * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few - * remaining bytes into a space-padded scratch buffer and finishes with the - * regular parse_string. This keeps the delicate escape/Unicode handling in one - * place (the proven parse_string) rather than duplicating it. - * - * Correctness relies on stage 1 having validated the string, i.e. there is an - * unescaped closing quote within [src, buf_end); that quote is therefore inside - * the copied scratch, so parse_string finds it without running off the scratch. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { - // Far from the end: identical to parse_string's loop. The guard uses - // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past - // buf_end; escape/Unicode look-aheads read within the string (before the - // closing quote, which is < buf_end), so they are in bounds here too. - // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: - // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate - // following bytes (+6 from the '\'), then (if it sees \u) another - // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the - // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the - // chunk start. The +12 margin ensures that even on kernels where - // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays - // in-bounds. The scratch fallback (3*PAD) is already safe. - while (src + SIMDJSON_PADDING + 12 <= buf_end) { - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - if (bs_quote.has_quote_first()) { - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - if (escape_char == 'u') { - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - // Within the final SIMDJSON_PADDING bytes: copy what remains into a - // space-padded scratch (spaces are neither quote nor backslash, so they do not - // disturb matching) and let the regular parser finish from there. The closing - // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in - // the chunk starting at some offset <= remaining and reads at most - // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. - // We size at 3x for a comfortable margin (the unicode look-ahead reads a few - // extra bytes past an escape). - uint8_t scratch[SIMDJSON_PADDING * 3]; - const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING - std::memset(scratch, ' ', sizeof(scratch)); - std::memcpy(scratch, src, remaining); - return parse_string(scratch, dst, allow_replacement); -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for westmere */ -/* including generic/stage2/tape_builder.h for westmere: #include */ -/* begin file generic/stage2/tape_builder.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - - -namespace simdjson { -namespace westmere { -namespace { -namespace stage2 { - -template -struct tape_builder_impl { - /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; - /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator &iter) noexcept; - - /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator &iter) noexcept; - /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator &iter) noexcept; - /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator &iter) noexcept; - - /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator &iter) noexcept; - /** - * Called when a key in a field is encountered. - * - * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array - * will be called after this with the field value. - */ - simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; - /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator &iter) noexcept; - /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator &iter) noexcept; - - /** - * Called when a string, number, boolean or null is found. - */ - simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; - /** - * Called when a string, number, boolean or null is found at the top level of a document (i.e. - * when there is no array or object and the entire document is a single string, number, boolean or - * null. - * - * This is separate from primitive() because simdjson's normal primitive parsing routines assume - * there is at least one more token after the value, which is only true in an array or object. - */ - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - -#if SIMDJSON_ENABLE_NAN_INF - simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, - // this returns a tape error on failure. - simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; -#endif - - /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; - - /** Next location to write to tape */ - tape_writer tape; -public: - simdjson_inline tape_builder_impl(dom::document &doc) noexcept; -private: - /** Next write location in the string buf for stage 2 parsing */ - uint8_t *current_string_buf_loc; - - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - simdjson_inline void start_container(json_iterator &iter) noexcept; - simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder_impl - -// Thin, non-templated entry so each architecture's stage2() keeps calling -// tape_builder::parse_document unchanged. It chooses the bounds-safe -// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the -// choice is a compile-time constant inside the walk: the padded path carries no -// extra branch or load (see tape_builder_impl::visit_string). -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } - - // The stage 2 entry points of dom::document_stream, which picks one per batch - // (see internal::dom_parser_implementation::stage2_next_function). Out of - // line, so that stage2_next() can share the code. - simdjson_warn_unused static simdjson_never_inline error_code parse_next_document( - internal::dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - return parse_document(static_cast(dom_parser), doc); - } - - // A document of a stream batch that stage 2 must bound to its structural - // indexes (see internal::dom_parser_implementation::_bounded_batch). - simdjson_warn_unused static simdjson_never_inline error_code parse_next_bounded_document( - internal::dom_parser_implementation &_dom_parser, dom::document &doc) noexcept { - dom_parser_implementation &dom_parser = static_cast(_dom_parser); - dom_parser.doc = &doc; - json_iterator iter(dom_parser, dom_parser.next_structural_index); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } -}; - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_root_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { - constexpr uint32_t start_tape_index = 0; - tape.append(start_tape_index, internal::tape_type::ROOT); - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { - return visit_string(iter, key, true); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 - return SUCCESS; -} - -template -simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { - iter.log_value(key ? "key" : "string"); - uint8_t *dst = on_start_string(iter); - // We do not allow replacement when the escape characters are invalid. - // UNPADDED is a compile-time constant chosen once per document by - // tape_builder::parse_document, so the padded build instantiates only the - // plain parse_string call below -- no runtime branch and no flag load. - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); - } else { - dst = stringparsing::parse_string(value+1, dst, false); - } - if (dst == nullptr) { - iter.log_error("Invalid escape in string"); - return STRING_ERROR; - } - on_end_string(dst); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { - return visit_string(iter, value); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("number"); - const uint8_t *num = value; - std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - // numberparsing reads ahead in 8-byte blocks for floats - // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a - // number whose digits reach the final bytes of an unpadded buffer would read - // past it. *(next_structural) is the offset of the token following this - // number, hence an upper bound on where the digits end; when that is within - // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail - // (mirroring visit_root_number). This fires only for numbers near the end. - if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { - const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document - copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, rl); - std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); - num = copy.get(); - } - } - error_code err = numberparsing::parse_number(num, tape); - if (simdjson_unlikely(err == BIGINT_ERROR && - iter.dom_parser._number_as_string)) { - // Write big integer to string buffer using the same format as strings. - // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = num; - if (*p == '-') p++; - while (numberparsing::is_digit(*p)) p++; - // The digit run must be terminated by a structural or whitespace character; otherwise the - // token is malformed (e.g. "123456789123456789123x"). - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - num); - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); - uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, num, len); - dst += len; - on_end_string(dst); - return SUCCESS; - } - return err; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { - // - // We need to make a copy to make sure that the string is space terminated. - // This is not about padding the input, which should already padded up - // to len + SIMDJSON_PADDING. However, we have no control at this stage - // on how the padding was done. What if the input string was padded with nulls? - // It is quite common for an input string to have an extra null character (C string). - // We do not want to allow 9\0 (where \0 is the null character) inside a JSON - // document, but the string "9\0" by itself is fine. So we make a copy and - // pad the input with spaces when we know that there is just one input element. - // This copy is relatively expensive, but it will almost never be called in - // practice unless you are in the strange scenario where you have many JSON - // documents made of single atoms. - // - // In a stream, the input goes on with other documents: copy up to the next - // structural only, not to the end of the batch. - const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); - std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, len); - std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); - error_code error = visit_number(iter, copy.get()); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated - // token at the very end of an unpadded buffer would over-read. Use the - // length-aware form there (the root variant already does this). - const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) - : atomparsing::is_valid_true_atom(value); - if (!ok) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) - : atomparsing::is_valid_false_atom(value); - if (!ok) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) - : atomparsing::is_valid_null_atom(value); - if (!ok) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -#if SIMDJSON_ENABLE_NAN_INF -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - // For unpadded input use the length-aware validator so the 'infinity'-style - // 8-byte compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) - : atomparsing::is_valid_nan_atom(value); - if (!ok) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. - // For unpadded input use the length-aware validator so the 'infinity' 8-byte - // compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) - : atomparsing::is_valid_inf_atom(value); - if (!ok) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure - if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -// private: - -template -simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { - return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - auto start_index = next_tape_index(iter); - tape.append(start_index+2, start); - tape.append(start_index, end); - return SUCCESS; -} - -template -simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); - iter.dom_parser.open_containers[iter.depth].count = 0; - tape.skip(); // We don't actually *write* the start element until the end. -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - // Write the ending tape element, pointing at the start location - const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; - tape.append(start_tape_index, end); - // Write the start tape element, pointing at the end location (and including count) - // count can overflow if it exceeds 24 bits... so we saturate - // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). - const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; - const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); - return SUCCESS; -} - -template -simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { - // we advance the point, accounting for the fact that we have a NULL termination - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); - return current_string_buf_loc + sizeof(uint32_t); -} - -template -simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { - uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); - // TODO check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); - // NULL termination is still handy if you expect all your strings to - // be NULL terminated? It comes at a small cost - *dst = 0; - current_string_buf_loc = dst + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H -/* end file generic/stage2/tape_builder.h for westmere */ -/* end file generic/stage2/amalgamated.h for westmere */ - -// -// Stage 1 -// - -namespace simdjson { -namespace westmere { - -simdjson_warn_unused error_code implementation::create_dom_parser_implementation( - size_t capacity, - size_t max_depth, - std::unique_ptr& dst -) const noexcept { - dst.reset( new (std::nothrow) dom_parser_implementation() ); - if (!dst) { return MEMALLOC; } - if (auto err = dst->set_capacity(capacity)) - return err; - if (auto err = dst->set_max_depth(max_depth)) - return err; - return SUCCESS; -} - -namespace { - -using namespace simd; - -simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why - // we can't use the generic lookup_16. - auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); - - // The 6 operators (:,[]{}) have these values: - // - // , 2C - // : 3A - // [ 5B - // { 7B - // ] 5D - // } 7D - // - // If you use | 0x20 to turn [ and ] into { and }, the lower 4 bits of each character is unique. - // We exploit this, using a simd 4-bit lookup to tell us which character match against, and then - // match it (against | 0x20). - // - // To prevent recognizing other characters, everything else gets compared with 0, which cannot - // match due to the | 0x20. - // - // NOTE: Due to the | 0x20, this ALSO treats and (control characters 0C and 1A) like , - // and :. This gets caught in stage 2, which checks the actual character to ensure the right - // operators are in the right places. - const auto op_table = simd8::repeat_16( - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, ':', '{', // : = 3A, [ = 5B, { = 7B - ',', '}', 0, 0 // , = 2C, ] = 5D, } = 7D - ); - - // We compute whitespace and op separately. If the code later only use one or the - // other, given the fact that all functions are aggressively inlined, we can - // hope that useless computations will be omitted. This is namely case when - // minifying (we only need whitespace). - - - const uint64_t whitespace = in.eq({ - _mm_shuffle_epi8(whitespace_table, in.chunks[0]), - _mm_shuffle_epi8(whitespace_table, in.chunks[1]), - _mm_shuffle_epi8(whitespace_table, in.chunks[2]), - _mm_shuffle_epi8(whitespace_table, in.chunks[3]) - }); - // Turn [ and ] into { and } - const simd8x64 curlified{ - in.chunks[0] | 0x20, - in.chunks[1] | 0x20, - in.chunks[2] | 0x20, - in.chunks[3] | 0x20 - }; - const uint64_t op = curlified.eq({ - _mm_shuffle_epi8(op_table, in.chunks[0]), - _mm_shuffle_epi8(op_table, in.chunks[1]), - _mm_shuffle_epi8(op_table, in.chunks[2]), - _mm_shuffle_epi8(op_table, in.chunks[3]) - }); - return { whitespace, op }; -} - -simdjson_inline bool is_ascii(const simd8x64& input) { - return input.reduce_or().is_ascii(); -} - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be >= 0x80 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be >= 0x80 - return is_third_byte | is_fourth_byte; -} - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -// -// Stage 2 -// - -// -// Implementation-specific overrides -// - -namespace simdjson { -namespace westmere { - -simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { - return westmere::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, stage1_mode streaming) noexcept { - this->buf = _buf; - this->len = _len; - return westmere::stage1::json_structural_indexer::index<64>(_buf, _len, *this, streaming); -} - -simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return westmere::stage1::generic_validate_utf8(buf,len); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_next_document(*this, _doc); -} - -internal::dom_parser_implementation::stage2_next_function_t dom_parser_implementation::stage2_next_function(bool bounded) const noexcept { - return bounded ? stage2::tape_builder::parse_next_bounded_document : stage2::tape_builder::parse_next_document; -} - -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { - return westmere::stringparsing::parse_string(src, dst, replacement_char); -} - -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { - return westmere::stringparsing::parse_wobbly_string(src, dst); -} - -simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { - auto error = stage1(_buf, _len, stage1_mode::regular); - if (error) { return error; } - return stage2(_doc); -} -} // namespace westmere -} // namespace simdjson - -/* including simdjson/westmere/end.h: #include */ -/* begin file simdjson/westmere/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_WESTMERE -SIMDJSON_UNTARGET_REGION -#endif - -/* undefining SIMDJSON_IMPLEMENTATION from "westmere" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/westmere/end.h */ - -#endif // SIMDJSON_SRC_WESTMERE_CPP -/* end file westmere.cpp */ -#endif -#if SIMDJSON_IMPLEMENTATION_LASX -/* including lasx.cpp: #include */ -/* begin file lasx.cpp */ -#ifndef SIMDJSON_SRC_LASX_CPP -#define SIMDJSON_SRC_LASX_CPP - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* including simdjson/lasx.h: #include */ -/* begin file simdjson/lasx.h */ -#ifndef SIMDJSON_LASX_H -#define SIMDJSON_LASX_H - -/* including simdjson/lasx/begin.h: #include "simdjson/lasx/begin.h" */ -/* begin file simdjson/lasx/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "lasx" */ -#define SIMDJSON_IMPLEMENTATION lasx -#include // This is a hack. We should not need to put this include here. -#if SIMDJSON_CAN_ALWAYS_RUN_LASX -// nothing needed. -#else -SIMDJSON_TARGET_REGION("lasx,lsx") -#endif - -/* including simdjson/lasx/base.h: #include "simdjson/lasx/base.h" */ -/* begin file simdjson/lasx/base.h */ -#ifndef SIMDJSON_LASX_BASE_H -#define SIMDJSON_LASX_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * Implementation for LASX. - */ -namespace lasx { - -class implementation; - -namespace { -namespace simd { -template struct simd8; -template struct simd8x64; -} // namespace simd -} // unnamed namespace - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_LASX_BASE_H -/* end file simdjson/lasx/base.h */ -/* including simdjson/lasx/intrinsics.h: #include "simdjson/lasx/intrinsics.h" */ -/* begin file simdjson/lasx/intrinsics.h */ -#ifndef SIMDJSON_LASX_INTRINSICS_H -#define SIMDJSON_LASX_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include - -static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for LoongArch ASX"); - -#endif // SIMDJSON_LASX_INTRINSICS_H -/* end file simdjson/lasx/intrinsics.h */ -/* including simdjson/lasx/bitmanipulation.h: #include "simdjson/lasx/bitmanipulation.h" */ -/* begin file simdjson/lasx/bitmanipulation.h */ -#ifndef SIMDJSON_LASX_BITMANIPULATION_H -#define SIMDJSON_LASX_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmask.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { - return __builtin_ctzll(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { - return __builtin_clzll(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int count_ones(uint64_t input_num) { - return __lasx_xvpickve2gr_w(__lasx_xvpcnt_d(__m256i(v4u64{input_num, 0, 0, 0})), 0); -} - - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_LASX_BITMANIPULATION_H -/* end file simdjson/lasx/bitmanipulation.h */ -/* including simdjson/lasx/bitmask.h: #include "simdjson/lasx/bitmask.h" */ -/* begin file simdjson/lasx/bitmask.h */ -#ifndef SIMDJSON_LASX_BITMASK_H -#define SIMDJSON_LASX_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; - return bitmask; -} - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif -/* end file simdjson/lasx/bitmask.h */ -/* including simdjson/lasx/numberparsing_defs.h: #include "simdjson/lasx/numberparsing_defs.h" */ -/* begin file simdjson/lasx/numberparsing_defs.h */ -#ifndef SIMDJSON_LASX_NUMBERPARSING_DEFS_H -#define SIMDJSON_LASX_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace lasx { -namespace numberparsing { - -// we don't have appropriate instructions, so let us use a scalar function -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - uint64_t val; - std::memcpy(&val, chars, sizeof(uint64_t)); - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); - return answer; -} - -} // namespace numberparsing -} // namespace lasx -} // namespace simdjson - -#ifndef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_IS_BIG_ENDIAN -#define SIMDJSON_SWAR_NUMBER_PARSING 0 -#else -#define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif -#endif - -#endif // SIMDJSON_LASX_NUMBERPARSING_DEFS_H -/* end file simdjson/lasx/numberparsing_defs.h */ -/* including simdjson/lasx/simd.h: #include "simdjson/lasx/simd.h" */ -/* begin file simdjson/lasx/simd.h */ -#ifndef SIMDJSON_LASX_SIMD_H -#define SIMDJSON_LASX_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m256i value; - - // Zero constructor - simdjson_inline base() : value{__m256i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m256i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m256i&() const { return this->value; } - simdjson_inline operator __m256i&() { return this->value; } - simdjson_inline operator const v32i8&() const { return (v32i8&)this->value; } - simdjson_inline operator v32i8&() { return (v32i8&)this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return __lasx_xvor_v(*this, other); } - simdjson_inline Child operator&(const Child other) const { return __lasx_xvand_v(*this, other); } - simdjson_inline Child operator^(const Child other) const { return __lasx_xvxor_v(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return __lasx_xvandn_v(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8: base> { - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m256i _value) : base>(_value) {} - - friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return __lasx_xvseq_b(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - __m256i hi = __lasx_xvbsll_v(*this, N); - __m256i lo = __lasx_xvbsrl_v(*this, 16 - N); - __m256i tmp = __lasx_xvbsrl_v(prev_chunk, 16 - N); - lo = __lasx_xvpermi_q(lo, tmp, 0x21); - return __lasx_xvor_v(hi, lo); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { return __lasx_xvreplgr2vr_b(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m256i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { - __m256i mask = __lasx_xvmskltz_b(*this); - return (__lasx_xvpickve2gr_w(mask, 4) << 16) | (__lasx_xvpickve2gr_w(mask, 0)); - } - simdjson_inline bool any() const { - __m256i v = __lasx_xvmsknz_b(*this); - return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); - } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { - return __lasx_xvreplgr2vr_b(_value); - } - static simdjson_inline simd8 zero() { return __lasx_xvldi(0); } - static simdjson_inline simd8 load(const T values[32]) { - return __lasx_xvld(reinterpret_cast(values), 0); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m256i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[32]) const { - return __lasx_xvst(*this, reinterpret_cast<__m256i *>(dst), 0); - } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return __lasx_xvadd_b(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return __lasx_xvsub_b(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return __lasx_xvshuf_b(lookup_table, lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - template - simdjson_inline void compress(uint32_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by haswell - // lasx do it in 4 steps, first 8 bytes and then second 8 bytes... - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // second significant 8 bits - uint8_t mask3 = uint8_t(mask >> 16); // ... - uint8_t mask4 = uint8_t(mask >> 24); // ... - // next line just loads the 64-bit values thintable_epi8[mask{1,2,3,4}] - // into a 256-bit register. - __m256i shufmask = {int64_t(thintable_epi8[mask1]), int64_t(thintable_epi8[mask2]) + 0x0808080808080808, int64_t(thintable_epi8[mask3]), int64_t(thintable_epi8[mask4]) + 0x0808080808080808}; - // this is the version "nearly pruned" - __m256i pruned = __lasx_xvshuf_b(*this, *this, shufmask); - // we still need to put the pieces back together. - // we compute the popcount of the first words: - int pop1 = BitsSetTable256mul2[mask1]; - int pop2 = BitsSetTable256mul2[mask2]; - int pop3 = BitsSetTable256mul2[mask3]; - - // then load the corresponding mask - __m256i masklo = __lasx_xvldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop1 * 8); - __m256i maskhi = __lasx_xvldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop3 * 8); - __m256i compactmask = __lasx_xvpermi_q(maskhi, masklo, 0x20); - __m256i answer = __lasx_xvshuf_b(pruned, pruned, compactmask); - __lasx_xvst(answer, reinterpret_cast(output), 0); - uint64_t value3 = __lasx_xvpickve2gr_du(answer, 2); - uint64_t value4 = __lasx_xvpickve2gr_du(answer, 3); - uint64_t *pos = reinterpret_cast(reinterpret_cast(output) + 16 - (pop1 + pop2) / 2); - pos[0] = value3; - pos[1] = value4; - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, - int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, - int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31 - ) : simd8({ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v16,v17,v18,v19,v20,v21,v22,v23, - v24,v25,v26,v27,v28,v29,v30,v31 - }) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return __lasx_xvmax_b(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return __lasx_xvmin_b(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return __lasx_xvslt_b(other, *this); } - simdjson_inline simd8 operator<(const simd8 other) const { return __lasx_xvslt_b(*this, other); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, - uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, - uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31 - ) : simd8(__m256i(v32u8{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v16,v17,v18,v19,v20,v21,v22,v23, - v24,v25,v26,v27,v28,v29,v30,v31 - })) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return __lasx_xvsadd_bu(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return __lasx_xvssub_bu(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return __lasx_xvmax_bu(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return __lasx_xvmin_bu(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { - __m256i mask = __lasx_xvmskltz_b(*this); - return (0 == __lasx_xvpickve2gr_w(mask, 0)) && (0 == __lasx_xvpickve2gr_w(mask, 4)); - } - simdjson_inline bool bits_not_set_anywhere() const { - __m256i v = __lasx_xvmsknz_b(*this); - return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); - } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { - __m256i v = __lasx_xvmsknz_b(__lasx_xvand_v(*this, bits)); - return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); - } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(__lasx_xvsrli_b(*this, N)); } - template - simdjson_inline simd8 shl() const { return simd8(__lasx_xvslli_b(*this, N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 2, "LASX kernel should use two registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+32)} {} - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - uint32_t mask1 = uint32_t(mask); - uint32_t mask2 = uint32_t(mask >> 32); - __m256i zcnt = __lasx_xvpcnt_w(__m256i(v4u64{~mask, 0, 0, 0})); - uint64_t zcnt1 = __lasx_xvpickve2gr_wu(zcnt, 0); - uint64_t zcnt2 = __lasx_xvpickve2gr_wu(zcnt, 1); - // There should be a critical value which processes in scaler is faster. - if (zcnt1) - this->chunks[0].compress(mask1, output); - if (zcnt2) - this->chunks[1].compress(mask2, output + zcnt1); - return zcnt1 + zcnt2; - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - } - - simdjson_inline uint64_t to_bitmask() const { - __m256i mask0 = __lasx_xvmskltz_b(this->chunks[0]); - __m256i mask1 = __lasx_xvmskltz_b(this->chunks[1]); - __m256i mask_tmp = __lasx_xvpickve_w(mask0, 4); - __m256i tmp = __lasx_xvpickve_w(mask1, 4); - mask0 = __lasx_xvinsve0_w(mask0, mask1, 1); - mask_tmp = __lasx_xvinsve0_w(mask_tmp, tmp, 1); - return __lasx_xvpickve2gr_du(__lasx_xvpackev_h(mask_tmp, mask0), 0); - } - - simdjson_inline simd8 reduce_or() const { - return this->chunks[0] | this->chunks[1]; - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_LASX_SIMD_H -/* end file simdjson/lasx/simd.h */ -/* including simdjson/lasx/stringparsing_defs.h: #include "simdjson/lasx/stringparsing_defs.h" */ -/* begin file simdjson/lasx/stringparsing_defs.h */ -#ifndef SIMDJSON_LASX_STRINGPARSING_DEFS_H -#define SIMDJSON_LASX_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - v.store(dst); - return { - static_cast((v == '\\').to_bitmask()), // bs_bits - static_cast((v == '"').to_bitmask()), // quote_bits - }; -} - - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_LASX_STRINGPARSING_DEFS_H -/* end file simdjson/lasx/stringparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 - - -/* end file simdjson/lasx/begin.h */ -/* including simdjson/generic/amalgamated.h for lasx: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for lasx */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for lasx: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for lasx */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for lasx */ -/* including simdjson/generic/jsoncharutils.h for lasx: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for lasx */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for lasx */ -/* including simdjson/generic/atomparsing.h for lasx: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for lasx */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace lasx { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for lasx */ -/* including simdjson/generic/dom_parser_implementation.h for lasx: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for lasx */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace lasx -} // namespace simdjson - -namespace simdjson { -namespace lasx { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for lasx */ -/* including simdjson/generic/implementation_simdjson_result_base.h for lasx: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for lasx */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for lasx */ -/* including simdjson/generic/numberparsing.h for lasx: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for lasx */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace lasx { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for lasx */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for lasx: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for lasx */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for lasx */ -/* end file simdjson/generic/amalgamated.h for lasx */ -/* including simdjson/lasx/end.h: #include "simdjson/lasx/end.h" */ -/* begin file simdjson/lasx/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT -/* undefining SIMDJSON_IMPLEMENTATION from "lasx" */ -#undef SIMDJSON_IMPLEMENTATION - - -#if SIMDJSON_CAN_ALWAYS_RUN_LASX -// nothing needed. -#else -SIMDJSON_UNTARGET_REGION -#endif -/* end file simdjson/lasx/end.h */ - -#endif // SIMDJSON_LASX_H -/* end file simdjson/lasx.h */ -/* including simdjson/lasx/implementation.h: #include */ -/* begin file simdjson/lasx/implementation.h */ -#ifndef SIMDJSON_LASX_IMPLEMENTATION_H -#define SIMDJSON_LASX_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation("lasx", "LoongArch ASX", internal::instruction_set::LASX) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_LASX_IMPLEMENTATION_H -/* end file simdjson/lasx/implementation.h */ - -/* including simdjson/lasx/begin.h: #include */ -/* begin file simdjson/lasx/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "lasx" */ -#define SIMDJSON_IMPLEMENTATION lasx -#include // This is a hack. We should not need to put this include here. -#if SIMDJSON_CAN_ALWAYS_RUN_LASX -// nothing needed. -#else -SIMDJSON_TARGET_REGION("lasx,lsx") -#endif - -/* including simdjson/lasx/base.h: #include "simdjson/lasx/base.h" */ -/* begin file simdjson/lasx/base.h */ -#ifndef SIMDJSON_LASX_BASE_H -#define SIMDJSON_LASX_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * Implementation for LASX. - */ -namespace lasx { - -class implementation; - -namespace { -namespace simd { -template struct simd8; -template struct simd8x64; -} // namespace simd -} // unnamed namespace - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_LASX_BASE_H -/* end file simdjson/lasx/base.h */ -/* including simdjson/lasx/intrinsics.h: #include "simdjson/lasx/intrinsics.h" */ -/* begin file simdjson/lasx/intrinsics.h */ -#ifndef SIMDJSON_LASX_INTRINSICS_H -#define SIMDJSON_LASX_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include - -static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for LoongArch ASX"); - -#endif // SIMDJSON_LASX_INTRINSICS_H -/* end file simdjson/lasx/intrinsics.h */ -/* including simdjson/lasx/bitmanipulation.h: #include "simdjson/lasx/bitmanipulation.h" */ -/* begin file simdjson/lasx/bitmanipulation.h */ -#ifndef SIMDJSON_LASX_BITMANIPULATION_H -#define SIMDJSON_LASX_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmask.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { - return __builtin_ctzll(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { - return __builtin_clzll(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int count_ones(uint64_t input_num) { - return __lasx_xvpickve2gr_w(__lasx_xvpcnt_d(__m256i(v4u64{input_num, 0, 0, 0})), 0); -} - - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_LASX_BITMANIPULATION_H -/* end file simdjson/lasx/bitmanipulation.h */ -/* including simdjson/lasx/bitmask.h: #include "simdjson/lasx/bitmask.h" */ -/* begin file simdjson/lasx/bitmask.h */ -#ifndef SIMDJSON_LASX_BITMASK_H -#define SIMDJSON_LASX_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; - return bitmask; -} - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif -/* end file simdjson/lasx/bitmask.h */ -/* including simdjson/lasx/numberparsing_defs.h: #include "simdjson/lasx/numberparsing_defs.h" */ -/* begin file simdjson/lasx/numberparsing_defs.h */ -#ifndef SIMDJSON_LASX_NUMBERPARSING_DEFS_H -#define SIMDJSON_LASX_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace lasx { -namespace numberparsing { - -// we don't have appropriate instructions, so let us use a scalar function -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - uint64_t val; - std::memcpy(&val, chars, sizeof(uint64_t)); - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); - return answer; -} - -} // namespace numberparsing -} // namespace lasx -} // namespace simdjson - -#ifndef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_IS_BIG_ENDIAN -#define SIMDJSON_SWAR_NUMBER_PARSING 0 -#else -#define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif -#endif - -#endif // SIMDJSON_LASX_NUMBERPARSING_DEFS_H -/* end file simdjson/lasx/numberparsing_defs.h */ -/* including simdjson/lasx/simd.h: #include "simdjson/lasx/simd.h" */ -/* begin file simdjson/lasx/simd.h */ -#ifndef SIMDJSON_LASX_SIMD_H -#define SIMDJSON_LASX_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m256i value; - - // Zero constructor - simdjson_inline base() : value{__m256i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m256i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m256i&() const { return this->value; } - simdjson_inline operator __m256i&() { return this->value; } - simdjson_inline operator const v32i8&() const { return (v32i8&)this->value; } - simdjson_inline operator v32i8&() { return (v32i8&)this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return __lasx_xvor_v(*this, other); } - simdjson_inline Child operator&(const Child other) const { return __lasx_xvand_v(*this, other); } - simdjson_inline Child operator^(const Child other) const { return __lasx_xvxor_v(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return __lasx_xvandn_v(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8: base> { - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m256i _value) : base>(_value) {} - - friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return __lasx_xvseq_b(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - __m256i hi = __lasx_xvbsll_v(*this, N); - __m256i lo = __lasx_xvbsrl_v(*this, 16 - N); - __m256i tmp = __lasx_xvbsrl_v(prev_chunk, 16 - N); - lo = __lasx_xvpermi_q(lo, tmp, 0x21); - return __lasx_xvor_v(hi, lo); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { return __lasx_xvreplgr2vr_b(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m256i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { - __m256i mask = __lasx_xvmskltz_b(*this); - return (__lasx_xvpickve2gr_w(mask, 4) << 16) | (__lasx_xvpickve2gr_w(mask, 0)); - } - simdjson_inline bool any() const { - __m256i v = __lasx_xvmsknz_b(*this); - return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); - } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { - return __lasx_xvreplgr2vr_b(_value); - } - static simdjson_inline simd8 zero() { return __lasx_xvldi(0); } - static simdjson_inline simd8 load(const T values[32]) { - return __lasx_xvld(reinterpret_cast(values), 0); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m256i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[32]) const { - return __lasx_xvst(*this, reinterpret_cast<__m256i *>(dst), 0); - } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return __lasx_xvadd_b(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return __lasx_xvsub_b(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return __lasx_xvshuf_b(lookup_table, lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - template - simdjson_inline void compress(uint32_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by haswell - // lasx do it in 4 steps, first 8 bytes and then second 8 bytes... - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // second significant 8 bits - uint8_t mask3 = uint8_t(mask >> 16); // ... - uint8_t mask4 = uint8_t(mask >> 24); // ... - // next line just loads the 64-bit values thintable_epi8[mask{1,2,3,4}] - // into a 256-bit register. - __m256i shufmask = {int64_t(thintable_epi8[mask1]), int64_t(thintable_epi8[mask2]) + 0x0808080808080808, int64_t(thintable_epi8[mask3]), int64_t(thintable_epi8[mask4]) + 0x0808080808080808}; - // this is the version "nearly pruned" - __m256i pruned = __lasx_xvshuf_b(*this, *this, shufmask); - // we still need to put the pieces back together. - // we compute the popcount of the first words: - int pop1 = BitsSetTable256mul2[mask1]; - int pop2 = BitsSetTable256mul2[mask2]; - int pop3 = BitsSetTable256mul2[mask3]; - - // then load the corresponding mask - __m256i masklo = __lasx_xvldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop1 * 8); - __m256i maskhi = __lasx_xvldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop3 * 8); - __m256i compactmask = __lasx_xvpermi_q(maskhi, masklo, 0x20); - __m256i answer = __lasx_xvshuf_b(pruned, pruned, compactmask); - __lasx_xvst(answer, reinterpret_cast(output), 0); - uint64_t value3 = __lasx_xvpickve2gr_du(answer, 2); - uint64_t value4 = __lasx_xvpickve2gr_du(answer, 3); - uint64_t *pos = reinterpret_cast(reinterpret_cast(output) + 16 - (pop1 + pop2) / 2); - pos[0] = value3; - pos[1] = value4; - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, - int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, - int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31 - ) : simd8({ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v16,v17,v18,v19,v20,v21,v22,v23, - v24,v25,v26,v27,v28,v29,v30,v31 - }) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return __lasx_xvmax_b(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return __lasx_xvmin_b(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return __lasx_xvslt_b(other, *this); } - simdjson_inline simd8 operator<(const simd8 other) const { return __lasx_xvslt_b(*this, other); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, - uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, - uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31 - ) : simd8(__m256i(v32u8{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v16,v17,v18,v19,v20,v21,v22,v23, - v24,v25,v26,v27,v28,v29,v30,v31 - })) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return __lasx_xvsadd_bu(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return __lasx_xvssub_bu(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return __lasx_xvmax_bu(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return __lasx_xvmin_bu(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { - __m256i mask = __lasx_xvmskltz_b(*this); - return (0 == __lasx_xvpickve2gr_w(mask, 0)) && (0 == __lasx_xvpickve2gr_w(mask, 4)); - } - simdjson_inline bool bits_not_set_anywhere() const { - __m256i v = __lasx_xvmsknz_b(*this); - return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); - } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { - __m256i v = __lasx_xvmsknz_b(__lasx_xvand_v(*this, bits)); - return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); - } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(__lasx_xvsrli_b(*this, N)); } - template - simdjson_inline simd8 shl() const { return simd8(__lasx_xvslli_b(*this, N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 2, "LASX kernel should use two registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+32)} {} - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - uint32_t mask1 = uint32_t(mask); - uint32_t mask2 = uint32_t(mask >> 32); - __m256i zcnt = __lasx_xvpcnt_w(__m256i(v4u64{~mask, 0, 0, 0})); - uint64_t zcnt1 = __lasx_xvpickve2gr_wu(zcnt, 0); - uint64_t zcnt2 = __lasx_xvpickve2gr_wu(zcnt, 1); - // There should be a critical value which processes in scaler is faster. - if (zcnt1) - this->chunks[0].compress(mask1, output); - if (zcnt2) - this->chunks[1].compress(mask2, output + zcnt1); - return zcnt1 + zcnt2; - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - } - - simdjson_inline uint64_t to_bitmask() const { - __m256i mask0 = __lasx_xvmskltz_b(this->chunks[0]); - __m256i mask1 = __lasx_xvmskltz_b(this->chunks[1]); - __m256i mask_tmp = __lasx_xvpickve_w(mask0, 4); - __m256i tmp = __lasx_xvpickve_w(mask1, 4); - mask0 = __lasx_xvinsve0_w(mask0, mask1, 1); - mask_tmp = __lasx_xvinsve0_w(mask_tmp, tmp, 1); - return __lasx_xvpickve2gr_du(__lasx_xvpackev_h(mask_tmp, mask0), 0); - } - - simdjson_inline simd8 reduce_or() const { - return this->chunks[0] | this->chunks[1]; - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_LASX_SIMD_H -/* end file simdjson/lasx/simd.h */ -/* including simdjson/lasx/stringparsing_defs.h: #include "simdjson/lasx/stringparsing_defs.h" */ -/* begin file simdjson/lasx/stringparsing_defs.h */ -#ifndef SIMDJSON_LASX_STRINGPARSING_DEFS_H -#define SIMDJSON_LASX_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - v.store(dst); - return { - static_cast((v == '\\').to_bitmask()), // bs_bits - static_cast((v == '"').to_bitmask()), // quote_bits - }; -} - - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_LASX_STRINGPARSING_DEFS_H -/* end file simdjson/lasx/stringparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 - - -/* end file simdjson/lasx/begin.h */ -/* including generic/amalgamated.h for lasx: #include */ -/* begin file generic/amalgamated.h for lasx */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) -#error generic/dependencies.h must be included before generic/amalgamated.h! -#endif - -/* including generic/base.h for lasx: #include */ -/* begin file generic/base.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { - -struct json_character_block; - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_BASE_H -/* end file generic/base.h for lasx */ -/* including generic/dom_parser_implementation.h for lasx: #include */ -/* begin file generic/dom_parser_implementation.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// Interface a dom parser implementation must fulfill -namespace simdjson { -namespace lasx { -namespace { - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); -simdjson_inline bool is_ascii(const simd8x64& input); - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file generic/dom_parser_implementation.h for lasx */ -/* including generic/json_character_block.h for lasx: #include */ -/* begin file generic/json_character_block.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { - -struct json_character_block { - static simdjson_inline json_character_block classify(const simd::simd8x64& in); - - simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_inline uint64_t op() const noexcept { return _op; } - simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } - - uint64_t _whitespace; - uint64_t _op; -}; - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H -/* end file generic/json_character_block.h for lasx */ -/* end file generic/amalgamated.h for lasx */ -/* including generic/stage1/amalgamated.h for lasx: #include */ -/* begin file generic/stage1/amalgamated.h for lasx */ -// Stuff other things depend on -/* including generic/stage1/base.h for lasx: #include */ -/* begin file generic/stage1/base.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace stage1 { - -class bit_indexer; -template -struct buf_block_reader; -struct json_block; -class json_minifier; -class json_scanner; -struct json_string_block; -class json_string_scanner; -class json_structural_indexer; - -} // namespace stage1 - -namespace utf8_validation { -struct utf8_checker; -} // namespace utf8_validation - -using utf8_validation::utf8_checker; - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H -/* end file generic/stage1/base.h for lasx */ -/* including generic/stage1/buf_block_reader.h for lasx: #include */ -/* begin file generic/stage1/buf_block_reader.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace lasx { -namespace { -namespace stage1 { - -// Walks through a buffer in block-sized increments, loading the last part with spaces -template -struct buf_block_reader { -public: - simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t *full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t *dst) const; - /** - * Like get_remainder, for input followed by SIMDJSON_PADDING readable bytes. - * - * get_remainder writes the block with a memset and a memcpy of the remaining - * length, and the vector loads that read it back cannot be forwarded from - * those narrower stores: they wait until the stores reach the cache, which - * costs tens of cycles on every document. Here the block is put together in - * vector registers, from loads that may read into the padding, and written - * with full-width stores that the loads can forward from. - */ - simdjson_inline size_t get_remainder_padded(uint8_t *dst) const; - simdjson_inline void advance(); -private: - const uint8_t *buf; - const size_t len; - const size_t lenminusstep; - size_t idx; -}; - -template -simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} - -template -simdjson_inline size_t buf_block_reader::block_index() { return idx; } - -template -simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; -} - -template -simdjson_inline const uint8_t *buf_block_reader::full_block() const { - return &buf[idx]; -} - -template -simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { - if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; -} - -// keep_mask[128 - n + i] is 0xFF for i < n and 0 otherwise. -alignas(64) static const uint8_t keep_mask[256] = { - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF -}; - -template -simdjson_inline size_t buf_block_reader::get_remainder_padded(uint8_t *dst) const { - static_assert(STEP_SIZE <= 128, "keep_mask covers blocks of up to 128 bytes"); - static_assert(sizeof(simd::simd8) <= SIMDJSON_PADDING, "a vector load at the last byte must stay in the padding"); - const size_t remaining = len - idx; - if (remaining == 0) { return 0; } - const simd::simd8 spaces = simd::simd8::splat(0x20); - for (size_t off = 0; off < STEP_SIZE; off += sizeof(simd::simd8)) { - simd::simd8 v = spaces; - if (off < remaining) { - // Starts before len, so it ends within the padding. - const simd::simd8 in = simd::simd8::load(buf + idx + off); - const simd::simd8 keep = simd::simd8::load(keep_mask + 128 - remaining + off); - v = spaces ^ ((in ^ spaces) & keep); - } - v.store(dst + off); - } - return remaining; -} - -template -simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for lasx */ -/* including generic/stage1/json_escape_scanner.h for lasx: #include */ -/* begin file generic/stage1/json_escape_scanner.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace stage1 { - -/** - * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). - */ -struct json_escape_scanner { - /** The actual escape characters (the backslashes themselves). */ - uint64_t next_is_escaped = 0ULL; - - struct escaped_and_escape { - /** - * Mask of escaped characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 0100100010100101000 - * n \ \ n \ \ - * ``` - */ - uint64_t escaped; - /** - * Mask of escape characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 1001000101001010001 - * \ \ \ \ \ \ \ - * ``` - */ - uint64_t escape; - }; - - /** - * Get a mask of both escape and escaped characters (the characters following a backslash). - * - * @param potential_escape A mask of the character that can escape others (but could be - * escaped itself). e.g. block.eq('\\') - */ - simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { - -#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT - if (!backslash) { return {next_escaped_without_backslashes(), 0}; } -#endif - - // | | Mask (shows characters instead of 1's) | Depth | Instructions | - // |--------------------------------|----------------------------------------|-------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | - // | | ` even odd even odd odd` | | | - // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) - // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) - // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) - // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () - // (*) this is not needed until the next iteration - uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); - uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); - uint64_t escape = escape_and_terminal_code & backslash; - this->next_is_escaped = escape >> 63; - return {escaped, escape}; - } - -private: - static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; - - simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { - uint64_t escaped = this->next_is_escaped; - this->next_is_escaped = 0; - return escaped; - } - - /** - * Returns a mask of the next escape characters (masking out escaped backslashes), along with - * any non-backslash escape codes. - * - * \n \\n \\\n \\\\n returns: - * \n \ \ \n \ \ - * 11 100 1011 10100 - * - * You are expected to mask out the first bit yourself if the previous block had a trailing - * escape. - * - * & the result with potential_escape to get just the escape characters. - * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. - */ - static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { - // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: - // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be - // inverted (\\\ would be 010 instead of 101). - // - // ``` - // string: | ____\\\\_\\\\_____ | - // maybe_escaped | ODD | \ \ \ \ | - // even-aligned ^^^ ^^^^ odd-aligned - // ``` - // - // Taking that into account, our basic strategy is: - // - // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for - // odd-aligned runs. - // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the - // odd bits in odd-aligned runs. - // 3. & with backslash to clean up any stray bits. - // runs are set to 0, and then XORing with "odd": - // - // | | Mask (shows characters instead of 1's) | Instructions | - // |--------------------------------|----------------------------------------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | - // | | ` even odd even odd odd` | - // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) - // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) - // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) - // - - // Escaped characters are characters following an escape. - uint64_t maybe_escaped = potential_escape << 1; - - // To distinguish odd from even escape sequences, therefore, we turn on any *starting* - // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) - // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. - // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. - // - All other odd bytes are 1, and even bytes are 0. - uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; - uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; - - // Now we flip all odd bytes back with xor. This: - // - Makes odd runs of backslashes go from 0000 to 1010 - // - Makes even runs of backslashes go from 1111 to 1010 - // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) - // - Resets all other bytes to 0 - return even_series_codes_and_odd_bits ^ ODD_BITS; - } -}; - -} // namespace stage1 -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_escape_scanner.h for lasx */ -/* including generic/stage1/json_string_scanner.h for lasx: #include */ -/* begin file generic/stage1/json_string_scanner.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace stage1 { - -struct json_string_block { - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : - _escaped(escaped), _quote(quote), _in_string(in_string) {} - - // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } - // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } - // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } - // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } - - // escaped characters (backslashed--does not include the hex characters after \u) - uint64_t _escaped; - // real quotes (non-escaped ones) - uint64_t _quote; - // string characters (includes start quote but not end quote) - uint64_t _in_string; -}; - -// Scans blocks for string characters, storing the state necessary to do so -class json_string_scanner { -public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); - -private: - // Scans for escape characters - json_escape_scanner escape_scanner{}; - // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). - uint64_t prev_in_string = 0ULL; -}; - -// -// Return a mask of all string characters plus end quotes. -// -// prev_escaped is overflow saying whether the next character is escaped. -// prev_in_string is overflow saying whether we're still in a string. -// -// Backslash sequences outside of quotes will be detected in stage 2. -// -simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = escape_scanner.next(backslash).escaped; - const uint64_t quote = in.eq('"') & ~escaped; - - // - // prefix_xor flips on bits inside the string (and flips off the end quote). - // - // Then we xor with prev_in_string: if we were in a string already, its effect is flipped - // (characters inside strings are outside, and characters outside strings are inside). - // - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - - // - // Check if we're still in a string at the end of the box so the next block will know - // - prev_in_string = uint64_t(static_cast(in_string) >> 63); - - // Use ^ to turn the beginning quote off, and the end quote on. - - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_string_block(escaped, quote, in_string); -} - -simdjson_really_inline error_code json_string_scanner::finish() { - if (prev_in_string) { - return UNCLOSED_STRING; - } - return SUCCESS; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_string_scanner.h for lasx */ -/* including generic/stage1/utf8_lookup4_algorithm.h for lasx: #include */ -/* begin file generic/stage1/utf8_lookup4_algorithm.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace utf8_validation { - -using namespace simd; - - simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { -// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) -// Bit 1 = Too Long (ASCII followed by continuation) -// Bit 2 = Overlong 3-byte -// Bit 4 = Surrogate -// Bit 5 = Overlong 2-byte -// Bit 7 = Two Continuations - constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ - // 11______ 11______ - constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ - constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ - constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ - constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ - constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ - constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ - // 11110100 101_____ - // 11110101 1001____ - // 11110101 101_____ - // 1111011_ 1001____ - // 1111011_ 101_____ - // 11111___ 1001____ - // 11111___ 101_____ - constexpr const uint8_t TOO_LARGE_1000 = 1<<6; - // 11110101 1000____ - // 1111011_ 1000____ - // 11111___ 1000____ - constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ - - const simd8 byte_1_high = prev1.shr<4>().lookup_16( - // 0_______ ________ - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - // 10______ ________ - TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, - // 1100____ ________ - TOO_SHORT | OVERLONG_2, - // 1101____ ________ - TOO_SHORT, - // 1110____ ________ - TOO_SHORT | OVERLONG_3 | SURROGATE, - // 1111____ ________ - TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 - ); - constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( - // ____0000 ________ - CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, - // ____0001 ________ - CARRY | OVERLONG_2, - // ____001_ ________ - CARRY, - CARRY, - - // ____0100 ________ - CARRY | TOO_LARGE, - // ____0101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____011_ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - - // ____1___ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____1101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000 - ); - const simd8 byte_2_high = input.shr<4>().lookup_16( - // ________ 0_______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - - // ________ 1000____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, - // ________ 1001____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, - // ________ 101_____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - - // ________ 11______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT - ); - return (byte_1_high & byte_1_low & byte_2_high); - } - simdjson_inline simd8 check_multibyte_lengths(const simd8 input, - const simd8 prev_input, const simd8 sc) { - simd8 prev2 = input.prev<2>(prev_input); - simd8 prev3 = input.prev<3>(prev_input); - simd8 must23 = must_be_2_3_continuation(prev2, prev3); - simd8 must23_80 = must23 & uint8_t(0x80); - return must23_80 ^ sc; - } - - // - // Return nonzero if there are incomplete multibyte characters at the end of the block: - // e.g. if there is a 4-byte character, but it's 3 bytes from the end. - // - simdjson_inline simd8 is_incomplete(const simd8 input) { - // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): - // ... 1111____ 111_____ 11______ -#if SIMDJSON_IMPLEMENTATION_ICELAKE || (SIMDJSON_IMPLEMENTATION_RVV_VLS && __riscv_v_fixed_vlen >= 512) - static const uint8_t max_array[64] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#else - static const uint8_t max_array[32] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#endif - const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); - return input.gt_bits(max_value); - } - - struct utf8_checker { - // If this is nonzero, there has been a UTF-8 error. - simd8 error; - // The last input we received - simd8 prev_input_block; - // Whether the last input we received was incomplete (used for ASCII fast path) - simd8 prev_incomplete; - - // - // Check whether the current bytes are valid UTF-8. - // - simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { - // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes - // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) - simd8 prev1 = input.prev<1>(prev_input); - simd8 sc = check_special_cases(input, prev1); - this->error |= check_multibyte_lengths(input, prev_input, sc); - } - - // The only problem that can happen at EOF is that a multibyte character is too short - // or a byte value too large in the last bytes: check_special_cases only checks for bytes - // too large in the first of two bytes. - simdjson_inline void check_eof() { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; - } - - simdjson_inline void check_next_input(const simd8x64& input) { - if(simdjson_likely(is_ascii(input))) { - this->error |= this->prev_incomplete; - } else { - // you might think that a for-loop would work, but under Visual Studio, it is not good enough. - static_assert((simd8x64::NUM_CHUNKS == 1) - ||(simd8x64::NUM_CHUNKS == 2) - || (simd8x64::NUM_CHUNKS == 4), - "We support one, two or four chunks per 64-byte block."); - SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 1) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 2) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 4) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - this->check_utf8_bytes(input.get<2>(), input.get<1>()); - this->check_utf8_bytes(input.get<3>(), input.get<2>()); - } - this->prev_incomplete = is_incomplete(input.get::NUM_CHUNKS-1>()); - this->prev_input_block = input.get::NUM_CHUNKS-1>(); - } - } - // do not forget to call check_eof! - simdjson_warn_unused simdjson_inline error_code errors() { - return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; - } - - }; // struct utf8_checker -} // namespace utf8_validation - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H -/* end file generic/stage1/utf8_lookup4_algorithm.h for lasx */ -/* including generic/stage1/json_scanner.h for lasx: #include */ -/* begin file generic/stage1/json_scanner.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace stage1 { - -/** - * A block of scanned json, with information on operators and scalars. - * - * We seek to identify pseudo-structural characters. Anything that is inside - * a string must be omitted (hence & ~_string.string_tail()). - * Otherwise, pseudo-structural characters come in two forms. - * 1. We have the structural characters ([,],{,},:, comma). The - * term 'structural character' is from the JSON RFC. - * 2. We have the 'scalar pseudo-structural characters'. - * Scalars are quotes, and any character except structural characters and white space. - * - * To identify the scalar pseudo-structural characters, we must look at what comes - * before them: it must be a space, a quote or a structural characters. - * Starting with simdjson v0.3, we identify them by - * negation: we identify everything that is followed by a non-quote scalar, - * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. - */ -struct json_block { -public: - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - - /** - * The start of structurals. - * In simdjson prior to v0.3, these were called the pseudo-structural characters. - **/ - simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } - /** All JSON whitespace (i.e. not in a string) */ - simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } - - // Helpers - - /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } - /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } - - // string and escape characters - json_string_block _string; - // whitespace, structural characters ('operators'), scalars - json_character_block _characters; - // whether the previous character was a scalar - uint64_t _follows_potential_nonquote_scalar; -private: - // Potential structurals (i.e. disregarding strings) - - /** - * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". - * They may reside inside a string. - **/ - simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } - /** - * The start of non-operator runs, like 123, true and "abc". - * It main reside inside a string. - **/ - simdjson_inline uint64_t potential_scalar_start() const noexcept { - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space - // then we know that it is irrelevant structurally. - return _characters.scalar() & ~follows_potential_scalar(); - } - /** - * Whether the given character is immediately after a non-operator like 123, true. - * The characters following a quote are not included. - */ - simdjson_inline uint64_t follows_potential_scalar() const noexcept { - // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character - // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a - // white space. - // It is understood that within quoted region, anything at all could be marked (irrelevant). - return _follows_potential_nonquote_scalar; - } -}; - -/** - * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. - * - * The scanner starts by calculating two distinct things: - * - string characters (taking \" into account) - * - structural characters or 'operators' ([]{},:, comma) - * and scalars (runs of non-operators like 123, true and "abc") - * - * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: - * in particular, the operator/scalar bit will find plenty of things that are actually part of - * strings. When we're done, json_block will fuse the two together by masking out tokens that are - * part of a string. - */ -class json_scanner { -public: - json_scanner() = default; - simdjson_inline json_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_warn_unused simdjson_inline error_code finish(); - -private: - // Whether the last character of the previous iteration is part of a scalar token - // (anything except whitespace or a structural character/'operator'). - uint64_t prev_scalar = 0ULL; - json_string_scanner string_scanner{}; -}; - - -// -// Check if the current character immediately follows a matching character. -// -// For example, this checks for quotes with backslashes in front of them: -// -// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); -// -simdjson_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; -} - -simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { - json_string_block strings = string_scanner.next(in); - // identifies the white-space and the structural characters - json_character_block characters = json_character_block::classify(in); - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). - // - // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) - // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential - // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we - // may need to add an extra check when parsing strings. - // - // Performance: there are many ways to skin this cat. - const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); - uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_block( - strings,// strings is a function-local object so either it moves or the copy is elided. - characters, - follows_nonquote_scalar - ); -} - -simdjson_warn_unused simdjson_inline error_code json_scanner::finish() { - return string_scanner.finish(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H -/* end file generic/stage1/json_scanner.h for lasx */ - -// All other declarations -/* including generic/stage1/find_next_document_index.h for lasx: #include */ -/* begin file generic/stage1/find_next_document_index.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace stage1 { - -/** - * This algorithm is used to quickly identify the last structural position that - * makes up a complete document. - * - * It does this by going backwards and finding the last *document boundary* (a - * place where one value follows another without a comma between them). If the - * last document (the characters after the boundary) has an equal number of - * start and end brackets, it is considered complete. - * - * Simply put, we iterate over the structural characters, starting from - * the end. We consider that we found the end of a JSON document when the - * first element of the pair is NOT one of these characters: '{' '[' ':' ',' - * and when the second element is NOT one of these characters: '}' ']' ':' ','. - * - * This simple comparison works most of the time, but it does not cover cases - * where the batch's structural indexes contain a perfect amount of documents. - * In such a case, we do not have access to the structural index which follows - * the last document, therefore, we do not have access to the second element in - * the pair, and that means we cannot identify the last document. To fix this - * issue, we keep a count of the open and closed curly/square braces we found - * while searching for the pair. When we find a pair AND the count of open and - * closed curly/square braces is the same, we know that we just passed a - * complete document, therefore the last json buffer location is the end of the - * batch. - */ -simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { - const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - const uint8_t e = parser.buf[len - 1]; - return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && - e != ' ' && e != '\t' && e != '\n' && e != '\r'; -} - -/** - * The structural index past the last one of a partial batch is the end of this - * window, not a token of it: the byte there is the first byte of the next - * window. A finished value must not be rejected because that byte is ',', ':', - * ']' or '}'. A last document that itself ends with '{', '[', ',' or ':' is - * invalid JSON, yet its brackets may balance: walking it, the On-Demand API - * could read the next batch as a value. We leave such a document to the next - * batch, so that the batch ends between two documents. - */ -simdjson_inline bool ends_with_unfinished_document(dom_parser_implementation &parser) { - const uint8_t last = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - return last == '{' || last == '[' || last == ',' || last == ':'; -} - -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { - // Variant: do not count separately, just figure out depth - if(parser.n_structural_indexes == 0) { return 0; } - auto arr_cnt = 0; - auto obj_cnt = 0; - for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { - auto idxb = parser.structural_indexes[i]; - switch (parser.buf[idxb]) { - case ':': - case ',': - continue; - case '}': - obj_cnt--; - continue; - case ']': - arr_cnt--; - continue; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - auto idxa = parser.structural_indexes[i - 1]; - switch (parser.buf[idxa]) { - case '{': - case '[': - case ':': - case ',': - continue; - } - // Last document is complete, so the next document will appear after! - if (!arr_cnt && !obj_cnt) { - return defer_last ? i : parser.n_structural_indexes; - } - // Last document is incomplete; mark the document at i + 1 as the next one - return i; - } - // If we made it to the end, we want to finish counting to see if we have a full document. - switch (parser.buf[parser.structural_indexes[0]]) { - case '}': - obj_cnt--; - break; - case ']': - arr_cnt--; - break; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - if (!arr_cnt && !obj_cnt) { - // We have a complete document. - return defer_last ? 0 : parser.n_structural_indexes; - } - return 0; -} - -/** - * Sentinel value returned to indicate a document started but didn't fit - * (CAPACITY error), as opposed to 0 which means no document content found - * (EMPTY). - */ -constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; - -/** - * For RFC 7464 JSON text sequences, filter RS from structural indexes and - * find batch boundaries. - * - * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. - * RS bytes appear in structural_indexes as they are classified as scalars. - * This function: - * 1. Scans structural_indexes to find and count RS positions - * 2. Filters RS out of structural_indexes in-place - * 3. Determines batch boundaries based on RS positions - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @param scan_len Offset past which no value start may be derived. Equals len - * unless stage 1 dropped a trailing unclosed string, whose bytes it - * never validated. - * @return The number of structural indexes to keep (after RS filtering), - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t find_next_document_index_json_sequence( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start, - size_t scan_len) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Phase 1: Scan structural_indexes to find RS positions and handle them. - // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS - // is already in structural_indexes (it's an operator). For scalars like numbers, - // the digit following RS is NOT in structural_indexes because the scanner sees - // RS as a scalar, making the digit a scalar continuation, not a start. - // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the - // actual value start position. - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_rs_pos = 0; - uint32_t rs_count = 0; - - for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { - const uint32_t pos = parser.structural_indexes[read_idx]; - if (parser.buf[pos] == 0x1E) { - // This is an RS character - find the actual JSON value start. - last_rs_pos = pos; - rs_count++; - // Skip past this RS and any whitespace *and any additional RSes* - // to locate the real value. Consecutive RSes are degenerate - // "empty records" per RFC 7464; we collapse them here. They do - // not always appear as separate entries in structural_indexes - // because the scanner groups runs of adjacent non-whitespace - // scalar bytes (including RS) into a single scalar start. - uint32_t value_start = pos + 1; - while (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { - value_start++; - } else if (c == 0x1E) { - // Collapsed empty record. Still count it so rs_count reflects - // the true number of record markers and last_rs_pos tracks - // the final one. - last_rs_pos = value_start; - rs_count++; - value_start++; - } else { - break; - } - } - // If the scanner emitted additional structurals inside the - // whitespace+RS run we just walked over (i.e., isolated RSes - // separated by whitespace), skip past them so we do not - // double-count or double-emit. - while (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] < value_start) { - read_idx++; - } - // Check if the value start is an operator (always present in - // scanner structural_indexes) or a scalar-like start (which may - // be missing from structural_indexes and must be added here). - // Note: '"' is NOT always in structural_indexes. The scanner - // classifies '"' as a scalar character and emits it as a - // structural only when it is a *scalar start* (preceded by - // whitespace or an operator). When '"' immediately follows an - // RS (which the scanner also classifies as scalar), it is - // treated as a scalar continuation and not emitted - so we - // must add it here just like any other scalar value. - if (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - const bool is_operator = - (c == '{' || c == '}' || c == '[' || c == ']' || - c == ':' || c == ','); - // If the next scanner structural is exactly at value_start, - // the scanner already emitted it (it followed whitespace) and - // we must not add a duplicate - a subsequent iteration will - // copy it into write_idx. - const bool already_emitted = - (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] == value_start); - if (!is_operator && !already_emitted) { - // Scalar value (number/true/false/null/string) - add its - // position since scanner missed it. - parser.structural_indexes[write_idx++] = value_start; - } - } - } else { - // Not RS, copy to output - parser.structural_indexes[write_idx++] = pos; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { - // Only RS markers here: the last one opens a record continuing past the - // window, so that is where the next batch resumes. - if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } - return 0; - } - if (rs_count == 0) { - // No RS found; for final batch, try generic boundary detection - return is_final ? find_next_document_index(parser) : 0; - } - - // Phase 2: Determine batch boundaries based on RS positions - - if (is_final) { - // Final batch: all documents are complete (last one ends at EOF). - // In json_sequence mode, RS markers define document boundaries, so all - // remaining structurals form complete documents. Return them all directly. - // (Calling find_next_document_index() would fail for scalar documents.) - return parser.n_structural_indexes; - } - - // Partial batch: need to find complete documents only. - // A document starting at an RS is complete if there is another RS after it. - next_batch_start = last_rs_pos; - - if (rs_count < 2) { - // Only one RS, so we have at most one document that may be incomplete. - // We cannot confirm it is complete without another RS. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least 2 RS markers. The last complete document ends before last_rs_pos. - - // Find the structural index cutoff: keep only structurals < last_rs_pos. - // Since we already filtered RS, all remaining structurals are valid. - // We iterate backward to find the last structural before last_rs_pos. - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_rs_pos) { - keep_count = i; - break; - } - } - - // No structurals before the last RS - no complete documents - if (keep_count == 0) { return 0; } - - // All documents before the last RS are complete by definition (the next RS - // confirms their end). No need to call find_next_document_index() which - // would fail for scalar documents like `1` or `"hello"`. - return keep_count; -} - -/** - * Filter comma-delimited documents by removing root-level commas from - * structural indexes. - * - * For comma-delimited format like `{...},{...},{...}`, we need to remove - * the commas that separate documents (depth 0) while preserving commas - * inside arrays and objects (depth > 0). - * - * After filtering, the structural indexes look like whitespace-delimited - * documents, so find_next_document_index() works unchanged. - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @return The number of structural indexes to keep, - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t filter_comma_delimited( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Track depth to identify root-level commas (depth 0) - int depth = 0; - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_root_comma_pos = 0; - uint32_t root_comma_count = 0; - - for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { - uint32_t idx = parser.structural_indexes[i]; - uint8_t c = parser.buf[idx]; - - switch (c) { - case '{': case '[': - depth++; - parser.structural_indexes[write_idx++] = idx; - break; - case '}': case ']': - depth--; - parser.structural_indexes[write_idx++] = idx; - break; - case ',': - if (depth == 0) { - // Root-level comma = document boundary, skip it - last_root_comma_pos = idx; - root_comma_count++; - continue; - } - parser.structural_indexes[write_idx++] = idx; - break; - default: - // Colons, scalars, etc. - parser.structural_indexes[write_idx++] = idx; - break; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { return 0; } - - if (is_final) { - // Final batch: use standard boundary detection on filtered indexes - return find_next_document_index(parser); - } - - // Partial batch: need to find complete documents only. - // A document ending with a root comma is complete. - if (root_comma_count == 0) { - // No root commas found; we cannot confirm any document is complete. - // The whole batch might be one incomplete document. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least one root comma. Documents before the last comma are complete. - next_batch_start = last_root_comma_pos + 1; - - // Find the structural index cutoff: keep only structurals < last_root_comma_pos - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_root_comma_pos) { - keep_count = i; - break; - } - } - - if (keep_count == 0) { return 0; } - - // Use standard boundary detection on the complete portion - parser.n_structural_indexes = keep_count; - return find_next_document_index(parser); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H -/* end file generic/stage1/find_next_document_index.h for lasx */ -/* including generic/stage1/json_minifier.h for lasx: #include */ -/* begin file generic/stage1/json_minifier.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace lasx { -namespace { -namespace stage1 { - -class json_minifier { -public: - template - static error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept; - -private: - simdjson_inline json_minifier(uint8_t *_dst) - : dst{_dst} - {} - template - simdjson_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_warn_unused simdjson_inline error_code finish(uint8_t *dst_start, size_t &dst_len); - json_scanner scanner{}; - uint8_t *dst; -}; - -simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { - uint64_t mask = block.whitespace(); - dst += in.compress(mask, dst); -} - -simdjson_warn_unused simdjson_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { - error_code error = scanner.finish(); - if (error) { dst_len = 0; return error; } - dst_len = dst - dst_start; - return SUCCESS; -} - -template<> -simdjson_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - simd::simd8x64 in_2(block_buf+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1); - this->next(in_2, block_2); - reader.advance(); -} - -template<> -simdjson_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - json_block block_1 = scanner.next(in_1); - this->next(block_buf, block_1); - reader.advance(); -} - -template -error_code json_minifier::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept { - buf_block_reader reader(buf, len); - json_minifier minifier(dst); - - // Index the first n-1 blocks - while (reader.has_full_block()) { - minifier.step(reader.full_block(), reader); - } - - // Index the last (remainder) block, padded with spaces - uint8_t block[STEP_SIZE]; - size_t remaining_bytes = reader.get_remainder(block); - if (remaining_bytes > 0) { - // We do not want to write directly to the output stream. Rather, we write - // to a local buffer (for safety). - uint8_t out_block[STEP_SIZE]; - uint8_t * const guarded_dst{minifier.dst}; - minifier.dst = out_block; - minifier.step(block, reader); - size_t to_write = minifier.dst - out_block; - // In some cases, we could be enticed to consider the padded spaces - // as part of the string. This is fine as long as we do not write more - // than we consumed. - if(to_write > remaining_bytes) { to_write = remaining_bytes; } - memcpy(guarded_dst, out_block, to_write); - minifier.dst = guarded_dst + to_write; - } - return minifier.finish(dst, dst_len); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H -/* end file generic/stage1/json_minifier.h for lasx */ -/* including generic/stage1/json_structural_indexer.h for lasx: #include */ -/* begin file generic/stage1/json_structural_indexer.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace lasx { -namespace { -namespace stage1 { - -class bit_indexer { -public: - uint32_t *tail; - - simdjson_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} - -#if SIMDJSON_PREFER_REVERSE_BITS - /** - * ARM lacks a fast trailing zero instruction, but it has a fast - * bit reversal instruction and a fast leading zero instruction. - * Thus it may be profitable to reverse the bits (once) and then - * to rely on a sequence of instructions that call the leading - * zero instruction. - * - * Performance notes: - * The chosen routine is not optimal in terms of data dependency - * since zero_leading_bit might require two instructions. However, - * it tends to minimize the total number of instructions which is - * beneficial. - */ - simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { - int lz = leading_zeroes(rev_bits); - this->tail[i] = static_cast(idx) + lz; - rev_bits = zero_leading_bit(rev_bits, lz); - } -#else - /** - * Under recent x64 systems, we often have both a fast trailing zero - * instruction and a fast 'clear-lower-bit' instruction so the following - * algorithm can be competitive. - */ - - simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { - this->tail[i] = idx + trailing_zeroes(bits); - bits = clear_lowest_bit(bits); - } -#endif // SIMDJSON_PREFER_REVERSE_BITS - - template - simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { - write_index(idx, bits, START); - SIMDJSON_IF_CONSTEXPR (N > 1) { - write_indexes<(N-1>0?START+1:START), (N-1>=0?N-1:1)>(idx, bits); - } - return START+N; - } - - template - simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { - write_indexes(idx, bits); - SIMDJSON_IF_CONSTEXPR ((START+STEP) < END) { - if (simdjson_unlikely((START+STEP) < cnt)) { - write_indexes_stepped<(START+STEP(idx, bits, cnt); - } - } - return ((END-START) % STEP) == 0 ? END : (END-START) - ((END-START) % STEP) + STEP; - } - - // flatten out values in 'bits' assuming that they are are to have values of idx - // plus their position in the bitvector, and store these indexes at - // base_ptr[base] incrementing base as we go - // will potentially store extra values beyond end of valid bits, so base_ptr - // needs to be large enough to handle this - // - // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it - // will provide its own version of the code. -#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - simdjson_inline void write(uint32_t idx, uint64_t bits); -#else - simdjson_inline void write(uint32_t idx, uint64_t bits) { - // In some instances, the next branch is expensive because it is mispredicted. - // Unfortunately, in other cases, - // it helps tremendously. - if (bits == 0) - return; - - int cnt = static_cast(count_ones(bits)); - -#if SIMDJSON_PREFER_REVERSE_BITS - bits = reverse_bits(bits); -#endif -#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP - static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; -#else - static constexpr const int STEP = 4; -#endif - static constexpr const int STEP_UNTIL = 24; - - write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); - SIMDJSON_IF_CONSTEXPR (STEP_UNTIL < 64) { - if (simdjson_unlikely(STEP_UNTIL < cnt)) { - for (int i=STEP_UNTIL; itail += cnt; - } -#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -}; - -class json_structural_indexer { -public: - /** - * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. - * - * @param partial Setting the partial parameter to true allows the find_structural_bits to - * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If - * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. - */ - template - static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept; - -private: - simdjson_inline json_structural_indexer(uint32_t *structural_indexes); - template - simdjson_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_warn_unused simdjson_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); - - json_scanner scanner{}; - utf8_checker checker{}; - bit_indexer indexer; - uint64_t prev_structurals = 0; - uint64_t unescaped_chars_error = 0; -}; - -simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} - -// Skip the last character if it is partial -simdjson_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { - if (simdjson_unlikely(len < 3)) { - switch (len) { - case 2: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left - return len; - case 1: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - return len; - case 0: - return len; - } - } - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left - return len; -} - -// -// PERF NOTES: -// We pipe 2 inputs through these stages: -// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load -// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. -// 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. -// The output of step 1 depends entirely on this information. These functions don't quite use -// up enough CPU: the second half of the functions is highly serial, only using 1 execution core -// at a time. The second input's scans has some dependency on the first ones finishing it, but -// they can make a lot of progress before they need that information. -// 3. Step 1 does not use enough capacity, so we run some extra stuff while we're waiting for that -// to finish: utf-8 checks and generating the output from the last iteration. -// -// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all -// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough -// workout. -// -template -error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept { - if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } - // We guard the rest of the code so that we can assume that len > 0 throughout. - if (len == 0) { return EMPTY; } - if (is_streaming(partial)) { - len = trim_partial_utf8(buf, len); - // If you end up with an empty window after trimming - // the partial UTF-8 bytes, then chances are good that you - // have an UTF-8 formatting error. - if(len == 0) { return UTF8_ERROR; } - } - buf_block_reader reader(buf, len); - json_structural_indexer indexer(parser.structural_indexes.get()); - - // Read all but the last block - while (reader.has_full_block()) { - indexer.step(reader.full_block(), reader); - } - // Take care of the last block (will always be there unless file is empty which is - // not supposed to happen.) - uint8_t block[STEP_SIZE]; - const size_t remaining = parser._unpadded ? reader.get_remainder(block) : reader.get_remainder_padded(block); - if (simdjson_unlikely(remaining == 0)) { return UNEXPECTED_ERROR; } - indexer.step(block, reader); - return indexer.finish(parser, reader.block_index(), len, partial); -} - -template<> -simdjson_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block); - simd::simd8x64 in_2(block+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1, reader.block_index()); - this->next(in_2, block_2, reader.block_index()+64); - reader.advance(); -} - -template<> -simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block); - json_block block_1 = scanner.next(in_1); - this->next(in_1, block_1, reader.block_index()); - reader.advance(); -} - -simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { - uint64_t unescaped = in.lteq(0x1F); -#if SIMDJSON_UTF8VALIDATION - checker.check_next_input(in); -#endif - indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser - prev_structurals = block.structural_start(); - unescaped_chars_error |= block.non_quote_inside_string(unescaped); -} - -// Drops the unclosed string that ends a json_sequence batch and leaves its -// start in structural_indexes[n_structural_indexes]. The last index is its -// opening quote, unless the quote directly follows an RS: the scanner reads -// RS as a scalar byte, so it indexes the RS and not the quote. Keep the RS. -simdjson_inline void drop_json_sequence_unclosed_string(dom_parser_implementation &parser, size_t len) noexcept { - uint32_t quote = parser.structural_indexes[parser.n_structural_indexes - 1]; - if (parser.buf[quote] == '"') { - parser.n_structural_indexes--; - return; - } - while (quote < len && parser.buf[quote] != '"') { quote++; } - parser.structural_indexes[parser.n_structural_indexes] = quote; -} - -simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial) { - // Write out the final iteration's structurals - indexer.write(uint32_t(idx-64), prev_structurals); - error_code error = scanner.finish(); - // We deliberately break down the next expression so that it is - // human readable. - const bool should_we_exit = is_streaming(partial) ? - ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING - : (error != SUCCESS); // if partial is false, we must have SUCCESS - const bool have_unclosed_string = (error == UNCLOSED_STRING); - parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); - /*** - * The On-Demand API requires special padding. - * - * This is related to https://github.com/simdjson/simdjson/issues/906 - * Basically, we want to make sure that if the parsing continues beyond the last (valid) - * structural character, it quickly stops. - * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. - * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing - * continues, then it must be [,] or }. - * Suppose it is ] or }. We backtrack to the first character, what could it be that would - * not trigger an error? It could be ] or } but no, because you can't start a document that way. - * It can't be a comma, a colon or any simple value. So the only way we could continue is - * if the repeated character is [. But if so, the document must start with [. But if the document - * starts with [, it should end with ]. If we enforce that rule, then we would get - * ][[ which is invalid. - * - * This is illustrated with the test array_iterate_unclosed_error() on the following input: - * R"({ "a": [,,)" - **/ - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final - parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); - parser.structural_indexes[parser.n_structural_indexes + 2] = 0; - parser.next_structural_index = 0; - - // Bail out only once the count and sentinels above are set: - // document_stream::truncated_bytes() reads them even on error. - if (simdjson_unlikely(should_we_exit)) { return error; } - if (unescaped_chars_error) { return UNESCAPED_CHARS; } - - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - return EMPTY; - } - if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { - return UNEXPECTED_ERROR; - } - if (partial == stage1_mode::streaming_partial) { - // If we have an unclosed string, then the last structural - // will be the quote and we want to make sure to omit it. - if(have_unclosed_string) { - parser.n_structural_indexes--; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - // We truncate the input to the end of the last complete document (or zero). - const bool partial_scalar = !have_unclosed_string && ends_with_partial_scalar(parser, len); - const bool unfinished = ends_with_unfinished_document(parser); - auto new_structural_indexes = find_next_document_index(parser, partial_scalar || unfinished); - if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { - // The batch holds a single document, whose brackets balance but which - // cannot be valid JSON: it is not too big, it is invalid. - if (unfinished && find_next_document_index(parser, partial_scalar) != 0) { return TAPE_ERROR; } - if(parser.structural_indexes[0] == 0) { - // If the buffer is partial and we started at index 0 but the document is - // incomplete, it's too big to parse. - return CAPACITY; - } else { - // It is possible that the document could be parsed, we just had a lot - // of white space. - parser.n_structural_indexes = 0; - return EMPTY; - } - } - parser.n_structural_indexes = new_structural_indexes; - } else if (partial == stage1_mode::streaming_final) { - if(have_unclosed_string) { parser.n_structural_indexes--; } - // We truncate the input to the end of the last complete document (or zero). - // Because partial == stage1_mode::streaming_final, it means that we may - // silently ignore trailing garbage. Though it sounds bad, we do it - // deliberately because many people who have streams of JSON documents - // will truncate them for processing. E.g., imagine that you are uncompressing - // the data from a size file or receiving it in chunks from the network. You - // may not know where exactly the last document will be. Meanwhile the - // document_stream instances allow people to know the JSON documents they are - // parsing (see the iterator.source() method). - parser.n_structural_indexes = find_next_document_index(parser); - // We store the initial n_structural_indexes so that the client can see - // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, - // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, - // otherwise, it will copy some prior index. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - // This next line is critical, do not change it unless you understand what you are - // doing. - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - // We tolerate an unclosed string at the very end of the stream. Indeed, users - // often load their data in bulk without being careful and they want us to ignore - // the trailing garbage. - return EMPTY; - } - } else if (partial == stage1_mode::json_sequence_partial) { - // RFC 7464: use RS positions for batch boundaries - // A discarded unclosed string also caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - // The next batch starts at an RS, which the On-Demand API never reads as a - // string, a container or a separator. But an invalid last document may make - // it skip that RS as a value: repeat it, so that the index past it is not - // the next record, which can be a string that does not close in this batch. - parser.structural_indexes[parser.n_structural_indexes + 1] = next_batch_start; - } else if (partial == stage1_mode::json_sequence_final) { - // RFC 7464: final batch, last document extends to EOF - // As above: a discarded unclosed string caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); - // The filter compacted structural_indexes in place and restored the EOF - // sentinel past the compacted end, so the copy below is either the start - // of a truncated document or len, as in streaming_final. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } else if (partial == stage1_mode::comma_delimited_partial) { - // Comma-delimited: filter root-level commas, use comma positions for batch boundaries - if(have_unclosed_string) { - parser.n_structural_indexes--; - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - } else if (partial == stage1_mode::comma_delimited_final) { - // Comma-delimited: final batch, last document extends to EOF - if(have_unclosed_string) { parser.n_structural_indexes--; } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } - checker.check_eof(); - return checker.errors(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. -#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H -/* end file generic/stage1/json_structural_indexer.h for lasx */ -/* including generic/stage1/utf8_validator.h for lasx: #include */ -/* begin file generic/stage1/utf8_validator.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace stage1 { - -/** - * Validates that the string is actual UTF-8. - */ -template -bool generic_validate_utf8(const uint8_t * input, size_t length) { - checker c{}; - buf_block_reader<64> reader(input, length); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - c.check_next_input(in); - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - c.check_next_input(in); - reader.advance(); - c.check_eof(); - return c.errors() == error_code::SUCCESS; -} - -bool generic_validate_utf8(const char * input, size_t length) { - return generic_validate_utf8(reinterpret_cast(input),length); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H -/* end file generic/stage1/utf8_validator.h for lasx */ -/* end file generic/stage1/amalgamated.h for lasx */ -/* including generic/stage2/amalgamated.h for lasx: #include */ -/* begin file generic/stage2/amalgamated.h for lasx */ -// Stuff other things depend on -/* including generic/stage2/base.h for lasx: #include */ -/* begin file generic/stage2/base.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace stage2 { - -class json_iterator; -struct tape_builder; -struct tape_writer; - -} // namespace stage2 -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H -/* end file generic/stage2/base.h for lasx */ -/* including generic/stage2/tape_writer.h for lasx: #include */ -/* begin file generic/stage2/tape_writer.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace lasx { -namespace { -namespace stage2 { - -struct tape_writer { - /** The next place to write to tape */ - uint64_t *next_tape_loc; - - /** Write a signed 64-bit value to tape. */ - simdjson_inline void append_s64(int64_t value) noexcept; - - /** Write an unsigned 64-bit value to tape. */ - simdjson_inline void append_u64(uint64_t value) noexcept; - - /** Write a double value to tape. */ - simdjson_inline void append_double(double value) noexcept; - - /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ - simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; - - /** - * Append a tape entry (an 8-bit type,and 56 bits worth of value). - */ - simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; - - /** - * Skip the current tape entry without writing. - * - * Used to skip the start of the container, since we'll come back later to fill it in when the - * container ends. - */ - simdjson_inline void skip() noexcept; - - /** - * Skip the number of tape entries necessary to write a large u64 or i64. - */ - simdjson_inline void skip_large_integer() noexcept; - - /** - * Skip the number of tape entries necessary to write a double. - */ - simdjson_inline void skip_double() noexcept; - - /** - * Write a value to a known location on tape. - * - * Used to go back and write out the start of a container after the container ends. - */ - simdjson_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; - -private: - /** - * Append both the tape entry, and a supplementary value following it. Used for types that need - * all 64 bits, such as double and uint64_t. - */ - template - simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; -}; // struct tape_writer - -simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { - append2(0, value, internal::tape_type::INT64); -} - -simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { - append(0, internal::tape_type::UINT64); - *next_tape_loc = value; - next_tape_loc++; -} - -/** Write a double value to tape. */ -simdjson_inline void tape_writer::append_double(double value) noexcept { - append2(0, value, internal::tape_type::DOUBLE); -} - -simdjson_inline void tape_writer::skip() noexcept { - next_tape_loc++; -} - -simdjson_inline void tape_writer::skip_large_integer() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::skip_double() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { - *next_tape_loc = val | ((uint64_t(char(t))) << 56); - next_tape_loc++; -} - -template -simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { - append(val, t); - static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); - memcpy(next_tape_loc, &val2, sizeof(val2)); - next_tape_loc++; -} - -simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { - tape_loc = val | ((uint64_t(char(t))) << 56); -} - -simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { - // Write to string buffer: [4-byte LE length][digits][null] - uint32_t str_len = uint32_t(len); - memcpy(string_buf, &str_len, sizeof(uint32_t)); - memcpy(string_buf + sizeof(uint32_t), src, len); - string_buf[sizeof(uint32_t) + len] = 0; - // Tape entry: offset into string buffer - // The caller must set the offset relative to doc.string_buf base - append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches - string_buf += sizeof(uint32_t) + len + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H -/* end file generic/stage2/tape_writer.h for lasx */ -/* including generic/stage2/logger.h for lasx: #include */ -/* begin file generic/stage2/logger.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - - -// This is for an internal-only stage 2 specific logger. -// Set LOG_ENABLED = true to log what stage 2 is doing! -namespace simdjson { -namespace lasx { -namespace { -namespace logger { - - static constexpr const char * DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; - -#if SIMDJSON_VERBOSE_LOGGING - static constexpr const bool LOG_ENABLED = true; -#else - static constexpr const bool LOG_ENABLED = false; -#endif - static constexpr const int LOG_EVENT_LEN = 20; - static constexpr const int LOG_BUFFER_LEN = 30; - static constexpr const int LOG_SMALL_BUFFER_LEN = 10; - static constexpr const int LOG_INDEX_LEN = 5; - - static int log_depth; // Not threadsafe. Log only. - - // Helper to turn unprintable or newline characters into spaces - static simdjson_inline char printable_char(char c) { - if (c >= 0x20) { - return c; - } else { - return ' '; - } - } - - // Print the header and set up log_start - static simdjson_inline void log_start() { - if (LOG_ENABLED) { - log_depth = 0; - printf("\n"); - printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); - printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN+2, DASHES, LOG_BUFFER_LEN+2, DASHES, LOG_SMALL_BUFFER_LEN+2, DASHES, 5+2, DASHES); - } - } - - simdjson_unused static simdjson_inline void log_string(const char *message) { - if (LOG_ENABLED) { - printf("%s\n", message); - } - } - - // Logs a single line from the stage 2 DOM parser - template - static simdjson_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { - if (LOG_ENABLED) { - printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); - auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; - auto next_index = structurals.next_structural; - auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); - auto next = &structurals.buf[*next_index]; - { - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i=0;i */ -/* begin file generic/stage2/json_iterator.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace stage2 { - -class json_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - uint32_t depth{0}; - - /** - * Walk the JSON document. - * - * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as - * the first parameter; some callbacks have other parameters as well: - * - * - visit_document_start() - at the beginning. - * - visit_document_end() - at the end (if things were successful). - * - * - visit_array_start() - at the start `[` of a non-empty array. - * - visit_array_end() - at the end `]` of a non-empty array. - * - visit_empty_array() - when an empty array is encountered. - * - * - visit_object_end() - at the start `]` of a non-empty object. - * - visit_object_start() - at the end `]` of a non-empty object. - * - visit_empty_object() - when an empty object is encountered. - * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is - * guaranteed to point at the first quote of the string (`"key"`). - * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. - * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. - * - * - increment_count(iter) - each time a value is found in an array or object. - */ - template - simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; - - /** - * Create an iterator capable of walking a JSON document. - * - * The document must have already passed through stage 1. - */ - simdjson_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); - - /** - * Look at the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *peek(const uint32_t *batch_end = nullptr) const noexcept; - /** - * Advance to the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *advance(const uint32_t *batch_end = nullptr) noexcept; - /** - * Get the remaining length of the document, from the start of the current token. - */ - simdjson_inline size_t remaining_len() const noexcept; - /** - * Check if we are at the end of the document. - * - * If this is true, there are no more tokens. - */ - simdjson_inline bool at_eof() const noexcept; - /** - * Check if we are at the beginning of the document. - */ - simdjson_inline bool at_beginning() const noexcept; - simdjson_inline uint8_t last_structural() const noexcept; - - /** - * Log that a value has been found. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_value(const char *type) const noexcept; - /** - * Log the start of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_start_value(const char *type) const noexcept; - /** - * Log the end of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_end_value(const char *type) const noexcept; - /** - * Log an error. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_error(const char *error) const noexcept; - - template - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; - template - simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; -}; - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { - logger::log_start(); - - // - // Start the document - // - if (at_eof()) { return EMPTY; } - // Only used when BOUNDED (see advance()); a local, so that it can stay in a - // register, and vanishes from the other walks. - const uint32_t *const batch_end = &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - log_start_value("document"); - SIMDJSON_TRY( visitor.visit_document_start(*this) ); - - // - // Read first value - // - { - auto value = advance(batch_end); - - // Make sure the outer object or array is closed before continuing; otherwise, there are ways we - // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 - if (!STREAMING) { - switch (*value) { - case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; - case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; - } - } - - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; - } - } - goto document_end; - -// -// Object parser states -// -object_begin: - log_start_value("object"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = false; - SIMDJSON_TRY( visitor.visit_object_start(*this) ); - - { - auto key = advance(batch_end); - if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.increment_count(*this) ); - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - -object_field: - if (simdjson_unlikely( (*advance(batch_end) != ':') )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -object_continue: - switch (*advance(batch_end)) { - case ',': - SIMDJSON_TRY( visitor.increment_count(*this) ); - { - auto key = advance(batch_end); - if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - goto object_field; - case '}': log_end_value("object"); SIMDJSON_TRY( visitor.visit_object_end(*this) ); goto scope_end; - default: log_error("No comma between object fields"); return TAPE_ERROR; - } - -scope_end: - depth--; - if (depth == 0) { goto document_end; } - if (dom_parser.is_array[depth]) { goto array_continue; } - goto object_continue; - -// -// Array parser states -// -array_begin: - log_start_value("array"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = true; - SIMDJSON_TRY( visitor.visit_array_start(*this) ); - SIMDJSON_TRY( visitor.increment_count(*this) ); - -array_value: - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -array_continue: - switch (*advance(batch_end)) { - case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; - case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; - default: log_error("Missing comma between array values"); return TAPE_ERROR; - } - -document_end: - log_end_value("document"); - SIMDJSON_TRY( visitor.visit_document_end(*this) ); - - dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); - - // If we didn't make it to the end, it's an error - if ( !STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes ) { - log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); - return TAPE_ERROR; - } - - return SUCCESS; - -} // walk_document() - -simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { -} - -// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires -// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). -// -// When BOUNDED (see dom_parser_implementation::_bounded_batch), a document may -// not run past batch_end, one past the last structural index of the batch: -// past it, we return the same dummy, and walk_document fails cleanly. -template -simdjson_inline const uint8_t *json_iterator::peek(const uint32_t *batch_end) const noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -template -simdjson_inline const uint8_t *json_iterator::advance(const uint32_t *batch_end) noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural++); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -simdjson_inline size_t json_iterator::remaining_len() const noexcept { - return dom_parser.len - *(next_structural-1); -} - -simdjson_inline bool json_iterator::at_eof() const noexcept { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; -} -simdjson_inline bool json_iterator::at_beginning() const noexcept { - return next_structural == dom_parser.structural_indexes.get(); -} -simdjson_inline uint8_t json_iterator::last_structural() const noexcept { - return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; -} - -simdjson_inline void json_iterator::log_value(const char *type) const noexcept { - logger::log_line(*this, "", type, ""); -} - -simdjson_inline void json_iterator::log_start_value(const char *type) const noexcept { - logger::log_line(*this, "+", type, ""); - if (logger::LOG_ENABLED) { logger::log_depth++; } -} - -simdjson_inline void json_iterator::log_end_value(const char *type) const noexcept { - if (logger::LOG_ENABLED) { logger::log_depth--; } - logger::log_line(*this, "-", type, ""); -} - -simdjson_inline void json_iterator::log_error(const char *error) const noexcept { - logger::log_line(*this, "", "ERROR", error); -} - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { - switch (*value) { - case '"': return visitor.visit_root_string(*this, value); - case 't': return visitor.visit_root_true_atom(*this, value); - case 'f': return visitor.visit_root_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_root_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_root_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_root_inf_atom(*this, value); -#else - case 'n': return visitor.visit_root_null_atom(*this, value); -#endif - case '-': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return visitor.visit_root_number(*this, value); - default: - log_error("Document starts with a non-value character"); - return TAPE_ERROR; - } -} -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { - // Use the fact that most scalars are going to be either strings or numbers. - if(*value == '"') { - return visitor.visit_string(*this, value); - } else if (((*value - '0') < 10) || (*value == '-')) { - return visitor.visit_number(*this, value); - } - // true, false, null are uncommon. - switch (*value) { - case 't': return visitor.visit_true_atom(*this, value); - case 'f': return visitor.visit_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_inf_atom(*this, value); -#else - case 'n': return visitor.visit_null_atom(*this, value); -#endif - default: - log_error("Non-value found when value was expected!"); - return TAPE_ERROR; - } -} - -} // namespace stage2 -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H -/* end file generic/stage2/json_iterator.h for lasx */ -/* including generic/stage2/stringparsing.h for lasx: #include */ -/* begin file generic/stage2/stringparsing.h for lasx */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace lasx { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - // Decode adjacent Unicode escapes without returning to the - // quote-and-backslash scanner between code points. - do { - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } while (src[0] == '\\' && src[1] == 'u'); - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -/** - * Bounds-safe variant of parse_string for input buffers that are NOT padded to - * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input - * byte (buf + len). It behaves exactly like parse_string while we are at least - * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays - * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few - * remaining bytes into a space-padded scratch buffer and finishes with the - * regular parse_string. This keeps the delicate escape/Unicode handling in one - * place (the proven parse_string) rather than duplicating it. - * - * Correctness relies on stage 1 having validated the string, i.e. there is an - * unescaped closing quote within [src, buf_end); that quote is therefore inside - * the copied scratch, so parse_string finds it without running off the scratch. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { - // Far from the end: identical to parse_string's loop. The guard uses - // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past - // buf_end; escape/Unicode look-aheads read within the string (before the - // closing quote, which is < buf_end), so they are in bounds here too. - // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: - // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate - // following bytes (+6 from the '\'), then (if it sees \u) another - // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the - // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the - // chunk start. The +12 margin ensures that even on kernels where - // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays - // in-bounds. The scratch fallback (3*PAD) is already safe. - while (src + SIMDJSON_PADDING + 12 <= buf_end) { - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - if (bs_quote.has_quote_first()) { - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - if (escape_char == 'u') { - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - // Within the final SIMDJSON_PADDING bytes: copy what remains into a - // space-padded scratch (spaces are neither quote nor backslash, so they do not - // disturb matching) and let the regular parser finish from there. The closing - // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in - // the chunk starting at some offset <= remaining and reads at most - // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. - // We size at 3x for a comfortable margin (the unicode look-ahead reads a few - // extra bytes past an escape). - uint8_t scratch[SIMDJSON_PADDING * 3]; - const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING - std::memset(scratch, ' ', sizeof(scratch)); - std::memcpy(scratch, src, remaining); - return parse_string(scratch, dst, allow_replacement); -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for lasx */ -/* including generic/stage2/tape_builder.h for lasx: #include */ -/* begin file generic/stage2/tape_builder.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - - -namespace simdjson { -namespace lasx { -namespace { -namespace stage2 { - -template -struct tape_builder_impl { - /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; - /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator &iter) noexcept; - - /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator &iter) noexcept; - /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator &iter) noexcept; - /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator &iter) noexcept; - - /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator &iter) noexcept; - /** - * Called when a key in a field is encountered. - * - * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array - * will be called after this with the field value. - */ - simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; - /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator &iter) noexcept; - /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator &iter) noexcept; - - /** - * Called when a string, number, boolean or null is found. - */ - simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; - /** - * Called when a string, number, boolean or null is found at the top level of a document (i.e. - * when there is no array or object and the entire document is a single string, number, boolean or - * null. - * - * This is separate from primitive() because simdjson's normal primitive parsing routines assume - * there is at least one more token after the value, which is only true in an array or object. - */ - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - -#if SIMDJSON_ENABLE_NAN_INF - simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, - // this returns a tape error on failure. - simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; -#endif - - /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; - - /** Next location to write to tape */ - tape_writer tape; -public: - simdjson_inline tape_builder_impl(dom::document &doc) noexcept; -private: - /** Next write location in the string buf for stage 2 parsing */ - uint8_t *current_string_buf_loc; - - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - simdjson_inline void start_container(json_iterator &iter) noexcept; - simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder_impl - -// Thin, non-templated entry so each architecture's stage2() keeps calling -// tape_builder::parse_document unchanged. It chooses the bounds-safe -// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the -// choice is a compile-time constant inside the walk: the padded path carries no -// extra branch or load (see tape_builder_impl::visit_string). -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } - - // The stage 2 entry points of dom::document_stream, which picks one per batch - // (see internal::dom_parser_implementation::stage2_next_function). Out of - // line, so that stage2_next() can share the code. - simdjson_warn_unused static simdjson_never_inline error_code parse_next_document( - internal::dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - return parse_document(static_cast(dom_parser), doc); - } - - // A document of a stream batch that stage 2 must bound to its structural - // indexes (see internal::dom_parser_implementation::_bounded_batch). - simdjson_warn_unused static simdjson_never_inline error_code parse_next_bounded_document( - internal::dom_parser_implementation &_dom_parser, dom::document &doc) noexcept { - dom_parser_implementation &dom_parser = static_cast(_dom_parser); - dom_parser.doc = &doc; - json_iterator iter(dom_parser, dom_parser.next_structural_index); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } -}; - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_root_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { - constexpr uint32_t start_tape_index = 0; - tape.append(start_tape_index, internal::tape_type::ROOT); - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { - return visit_string(iter, key, true); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 - return SUCCESS; -} - -template -simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { - iter.log_value(key ? "key" : "string"); - uint8_t *dst = on_start_string(iter); - // We do not allow replacement when the escape characters are invalid. - // UNPADDED is a compile-time constant chosen once per document by - // tape_builder::parse_document, so the padded build instantiates only the - // plain parse_string call below -- no runtime branch and no flag load. - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); - } else { - dst = stringparsing::parse_string(value+1, dst, false); - } - if (dst == nullptr) { - iter.log_error("Invalid escape in string"); - return STRING_ERROR; - } - on_end_string(dst); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { - return visit_string(iter, value); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("number"); - const uint8_t *num = value; - std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - // numberparsing reads ahead in 8-byte blocks for floats - // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a - // number whose digits reach the final bytes of an unpadded buffer would read - // past it. *(next_structural) is the offset of the token following this - // number, hence an upper bound on where the digits end; when that is within - // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail - // (mirroring visit_root_number). This fires only for numbers near the end. - if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { - const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document - copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, rl); - std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); - num = copy.get(); - } - } - error_code err = numberparsing::parse_number(num, tape); - if (simdjson_unlikely(err == BIGINT_ERROR && - iter.dom_parser._number_as_string)) { - // Write big integer to string buffer using the same format as strings. - // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = num; - if (*p == '-') p++; - while (numberparsing::is_digit(*p)) p++; - // The digit run must be terminated by a structural or whitespace character; otherwise the - // token is malformed (e.g. "123456789123456789123x"). - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - num); - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); - uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, num, len); - dst += len; - on_end_string(dst); - return SUCCESS; - } - return err; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { - // - // We need to make a copy to make sure that the string is space terminated. - // This is not about padding the input, which should already padded up - // to len + SIMDJSON_PADDING. However, we have no control at this stage - // on how the padding was done. What if the input string was padded with nulls? - // It is quite common for an input string to have an extra null character (C string). - // We do not want to allow 9\0 (where \0 is the null character) inside a JSON - // document, but the string "9\0" by itself is fine. So we make a copy and - // pad the input with spaces when we know that there is just one input element. - // This copy is relatively expensive, but it will almost never be called in - // practice unless you are in the strange scenario where you have many JSON - // documents made of single atoms. - // - // In a stream, the input goes on with other documents: copy up to the next - // structural only, not to the end of the batch. - const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); - std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, len); - std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); - error_code error = visit_number(iter, copy.get()); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated - // token at the very end of an unpadded buffer would over-read. Use the - // length-aware form there (the root variant already does this). - const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) - : atomparsing::is_valid_true_atom(value); - if (!ok) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) - : atomparsing::is_valid_false_atom(value); - if (!ok) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) - : atomparsing::is_valid_null_atom(value); - if (!ok) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -#if SIMDJSON_ENABLE_NAN_INF -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - // For unpadded input use the length-aware validator so the 'infinity'-style - // 8-byte compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) - : atomparsing::is_valid_nan_atom(value); - if (!ok) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. - // For unpadded input use the length-aware validator so the 'infinity' 8-byte - // compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) - : atomparsing::is_valid_inf_atom(value); - if (!ok) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure - if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -// private: - -template -simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { - return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - auto start_index = next_tape_index(iter); - tape.append(start_index+2, start); - tape.append(start_index, end); - return SUCCESS; -} - -template -simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); - iter.dom_parser.open_containers[iter.depth].count = 0; - tape.skip(); // We don't actually *write* the start element until the end. -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - // Write the ending tape element, pointing at the start location - const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; - tape.append(start_tape_index, end); - // Write the start tape element, pointing at the end location (and including count) - // count can overflow if it exceeds 24 bits... so we saturate - // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). - const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; - const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); - return SUCCESS; -} - -template -simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { - // we advance the point, accounting for the fact that we have a NULL termination - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); - return current_string_buf_loc + sizeof(uint32_t); -} - -template -simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { - uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); - // TODO check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); - // NULL termination is still handy if you expect all your strings to - // be NULL terminated? It comes at a small cost - *dst = 0; - current_string_buf_loc = dst + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H -/* end file generic/stage2/tape_builder.h for lasx */ -/* end file generic/stage2/amalgamated.h for lasx */ - -// -// Stage 1 -// -namespace simdjson { -namespace lasx { - -simdjson_warn_unused error_code implementation::create_dom_parser_implementation( - size_t capacity, - size_t max_depth, - std::unique_ptr& dst -) const noexcept { - dst.reset( new (std::nothrow) dom_parser_implementation() ); - if (!dst) { return MEMALLOC; } - if (auto err = dst->set_capacity(capacity)) - return err; - if (auto err = dst->set_max_depth(max_depth)) - return err; - return SUCCESS; -} - -namespace { - -using namespace simd; - -simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - // Inspired by haswell. - // LASX use low 5 bits as index. For the 6 operators (:,[]{}), the unique-5bits is [6:2]. - // The ASCII white-space and operators have these values: (char, hex, unique-5bits) - // (' ', 20, 00000) ('\t', 09, 01001) ('\n', 0A, 01010) ('\r', 0D, 01101) - // (',', 2C, 01011) (':', 3A, 01110) ('[', 5B, 10110) ('{', 7B, 11110) (']', 5D, 10111) ('}', 7D, 11111) - const simd8 ws_table = simd8::repeat_16( - ' ', 0, 0, 0, 0, 0, 0, 0, 0, '\t', '\n', 0, 0, '\r', 0, 0 - ); - const simd8 op_table_lo = simd8::repeat_16( - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ',', 0, 0, ':', 0 - ); - const simd8 op_table_hi = simd8::repeat_16( - 0, 0, 0, 0, 0, 0, '[', ']', 0, 0, 0, 0, 0, 0, '{', '}' - ); - uint64_t ws = in.eq({ - in.chunks[0].lookup_16(ws_table), - in.chunks[1].lookup_16(ws_table), - }); - uint64_t op = in.eq({ - __lasx_xvshuf_b(op_table_hi, op_table_lo, in.chunks[0].shr<2>()), - __lasx_xvshuf_b(op_table_hi, op_table_lo, in.chunks[1].shr<2>()), - }); - - return { ws, op }; -} - -simdjson_inline bool is_ascii(const simd8x64& input) { - return input.reduce_or().is_ascii(); -} - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be >= 0x80 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be >= 0x80 - return is_third_byte | is_fourth_byte; -} - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -// -// Stage 2 -// - -// -// Implementation-specific overrides -// -namespace simdjson { -namespace lasx { - -simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { - return lasx::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, stage1_mode streaming) noexcept { - this->buf = _buf; - this->len = _len; - return lasx::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); -} - -simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return lasx::stage1::generic_validate_utf8(buf,len); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_next_document(*this, _doc); -} - -internal::dom_parser_implementation::stage2_next_function_t dom_parser_implementation::stage2_next_function(bool bounded) const noexcept { - return bounded ? stage2::tape_builder::parse_next_bounded_document : stage2::tape_builder::parse_next_document; -} - -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { - return lasx::stringparsing::parse_string(src, dst, allow_replacement); -} - -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { - return lasx::stringparsing::parse_wobbly_string(src, dst); -} - -simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { - auto error = stage1(_buf, _len, stage1_mode::regular); - if (error) { return error; } - return stage2(_doc); -} - -} // namespace lasx -} // namespace simdjson - -/* including simdjson/lasx/end.h: #include */ -/* begin file simdjson/lasx/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT -/* undefining SIMDJSON_IMPLEMENTATION from "lasx" */ -#undef SIMDJSON_IMPLEMENTATION - - -#if SIMDJSON_CAN_ALWAYS_RUN_LASX -// nothing needed. -#else -SIMDJSON_UNTARGET_REGION -#endif -/* end file simdjson/lasx/end.h */ - -#endif // SIMDJSON_SRC_LASX_CPP -/* end file lasx.cpp */ -#endif -#if SIMDJSON_IMPLEMENTATION_LSX -/* including lsx.cpp: #include */ -/* begin file lsx.cpp */ -#ifndef SIMDJSON_SRC_LSX_CPP -#define SIMDJSON_SRC_LSX_CPP - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* including simdjson/lsx.h: #include */ -/* begin file simdjson/lsx.h */ -#ifndef SIMDJSON_LSX_H -#define SIMDJSON_LSX_H - -/* including simdjson/lsx/begin.h: #include "simdjson/lsx/begin.h" */ -/* begin file simdjson/lsx/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "lsx" */ -#define SIMDJSON_IMPLEMENTATION lsx -/* including simdjson/lsx/base.h: #include "simdjson/lsx/base.h" */ -/* begin file simdjson/lsx/base.h */ -#ifndef SIMDJSON_LSX_BASE_H -#define SIMDJSON_LSX_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * Implementation for LSX. - */ -namespace lsx { - -class implementation; - -namespace { -namespace simd { -template struct simd8; -template struct simd8x64; -} // namespace simd -} // unnamed namespace - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_LSX_BASE_H -/* end file simdjson/lsx/base.h */ -/* including simdjson/lsx/intrinsics.h: #include "simdjson/lsx/intrinsics.h" */ -/* begin file simdjson/lsx/intrinsics.h */ -#ifndef SIMDJSON_LSX_INTRINSICS_H -#define SIMDJSON_LSX_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for LoongArch SX"); - -#endif // SIMDJSON_LSX_INTRINSICS_H -/* end file simdjson/lsx/intrinsics.h */ -/* including simdjson/lsx/bitmanipulation.h: #include "simdjson/lsx/bitmanipulation.h" */ -/* begin file simdjson/lsx/bitmanipulation.h */ -#ifndef SIMDJSON_LSX_BITMANIPULATION_H -#define SIMDJSON_LSX_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmask.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { - return __builtin_ctzll(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { - return __builtin_clzll(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int count_ones(uint64_t input_num) { - return __lsx_vpickve2gr_w(__lsx_vpcnt_d(__m128i(v2u64{input_num, 0})), 0); -} - - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_LSX_BITMANIPULATION_H -/* end file simdjson/lsx/bitmanipulation.h */ -/* including simdjson/lsx/bitmask.h: #include "simdjson/lsx/bitmask.h" */ -/* begin file simdjson/lsx/bitmask.h */ -#ifndef SIMDJSON_LSX_BITMASK_H -#define SIMDJSON_LSX_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; - return bitmask; -} - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif -/* end file simdjson/lsx/bitmask.h */ -/* including simdjson/lsx/numberparsing_defs.h: #include "simdjson/lsx/numberparsing_defs.h" */ -/* begin file simdjson/lsx/numberparsing_defs.h */ -#ifndef SIMDJSON_LSX_NUMBERPARSING_DEFS_H -#define SIMDJSON_LSX_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace lsx { -namespace numberparsing { - -// we don't have appropriate instructions, so let us use a scalar function -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - uint64_t val; - std::memcpy(&val, chars, sizeof(uint64_t)); - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); - return answer; -} - -} // namespace numberparsing -} // namespace lsx -} // namespace simdjson - -#ifndef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_IS_BIG_ENDIAN -#define SIMDJSON_SWAR_NUMBER_PARSING 0 -#else -#define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif -#endif - -#endif // SIMDJSON_LSX_NUMBERPARSING_DEFS_H -/* end file simdjson/lsx/numberparsing_defs.h */ -/* including simdjson/lsx/simd.h: #include "simdjson/lsx/simd.h" */ -/* begin file simdjson/lsx/simd.h */ -#ifndef SIMDJSON_LSX_SIMD_H -#define SIMDJSON_LSX_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{__m128i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i&() const { return this->value; } - simdjson_inline operator __m128i&() { return this->value; } - simdjson_inline operator const v16i8&() const { return (v16i8&)this->value; } - simdjson_inline operator v16i8&() { return (v16i8&)this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return __lsx_vor_v(*this, other); } - simdjson_inline Child operator&(const Child other) const { return __lsx_vand_v(*this, other); } - simdjson_inline Child operator^(const Child other) const { return __lsx_vxor_v(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return __lsx_vandn_v(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8: base> { - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return __lsx_vseq_b(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return __lsx_vor_v(__lsx_vbsll_v(*this, N), __lsx_vbsrl_v(prev_chunk, 16 - N)); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { - return __lsx_vreplgr2vr_b(uint8_t(-(!!_value))); - } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return __lsx_vpickve2gr_w(__lsx_vmskltz_b(*this), 0); } - simdjson_inline bool any() const { return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(*this), 0); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { return __lsx_vreplgr2vr_b(_value); } - static simdjson_inline simd8 zero() { return __lsx_vldi(0); } - static simdjson_inline simd8 load(const T values[16]) { - return __lsx_vld(reinterpret_cast(values), 0); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { - return __lsx_vst(*this, reinterpret_cast<__m128i *>(dst), 0); - } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return __lsx_vadd_b(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return __lsx_vsub_b(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return __lsx_vshuf_b(lookup_table, lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - template - simdjson_inline void compress(uint16_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by haswell - // lsx do it in 2 steps, first 8 bytes and then second 8 bytes... - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register. - __m128i shufmask = {int64_t(thintable_epi8[mask1]), int64_t(thintable_epi8[mask2]) + 0x0808080808080808}; - // this is the version "nearly pruned" - __m128i pruned = __lsx_vshuf_b(*this, *this, shufmask); - // we still need to put the pieces back together. - // we compute the popcount of the first words: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask - __m128i compactmask = __lsx_vldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop1 * 8); - __m128i answer = __lsx_vshuf_b(pruned, pruned, compactmask); - __lsx_vst(answer, reinterpret_cast(output), 0); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[16]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8({ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return __lsx_vmax_b(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return __lsx_vmin_b(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return __lsx_vslt_b(other, *this); } - simdjson_inline simd8 operator<(const simd8 other) const { return __lsx_vslt_b(*this, other); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(__m128i(v16u8{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - })) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return __lsx_vsadd_bu(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return __lsx_vssub_bu(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return __lsx_vmax_bu(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return __lsx_vmin_bu(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return 0 == __lsx_vpickve2gr_w(__lsx_vmskltz_b(*this), 0); } - simdjson_inline bool bits_not_set_anywhere() const { return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(*this), 0); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { - return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(__lsx_vand_v(*this, bits)), 0); - } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(__lsx_vsrli_b(*this, N)); } - template - simdjson_inline simd8 shl() const { return simd8(__lsx_vslli_b(*this, N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "LSX kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - uint16_t mask1 = uint16_t(mask); - uint16_t mask2 = uint16_t(mask >> 16); - uint16_t mask3 = uint16_t(mask >> 32); - uint16_t mask4 = uint16_t(mask >> 48); - __m128i zcnt = __lsx_vpcnt_h(__m128i(v2u64{~mask, 0})); - uint64_t zcnt1 = __lsx_vpickve2gr_hu(zcnt, 0); - uint64_t zcnt2 = __lsx_vpickve2gr_hu(zcnt, 1); - uint64_t zcnt3 = __lsx_vpickve2gr_hu(zcnt, 2); - uint64_t zcnt4 = __lsx_vpickve2gr_hu(zcnt, 3); - uint8_t *voutput = reinterpret_cast(output); - // There should be a critical value which processes in scaler is faster. - if (zcnt1) - this->chunks[0].compress(mask1, reinterpret_cast(voutput)); - voutput += zcnt1; - if (zcnt2) - this->chunks[1].compress(mask2, reinterpret_cast(voutput)); - voutput += zcnt2; - if (zcnt3) - this->chunks[2].compress(mask3, reinterpret_cast(voutput)); - voutput += zcnt3; - if (zcnt4) - this->chunks[3].compress(mask4, reinterpret_cast(voutput)); - voutput += zcnt4; - return reinterpret_cast(voutput) - reinterpret_cast(output); - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - this->chunks[2].store(ptr+sizeof(simd8)*2); - this->chunks[3].store(ptr+sizeof(simd8)*3); - } - - simdjson_inline uint64_t to_bitmask() const { - __m128i mask1 = __lsx_vmskltz_b(this->chunks[0]); - __m128i mask2 = __lsx_vmskltz_b(this->chunks[1]); - __m128i mask3 = __lsx_vmskltz_b(this->chunks[2]); - __m128i mask4 = __lsx_vmskltz_b(this->chunks[3]); - mask1 = __lsx_vilvl_h(mask2, mask1); - mask2 = __lsx_vilvl_h(mask4, mask3); - return __lsx_vpickve2gr_du(__lsx_vilvl_w(mask2, mask1), 0); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_LSX_SIMD_H -/* end file simdjson/lsx/simd.h */ -/* including simdjson/lsx/stringparsing_defs.h: #include "simdjson/lsx/stringparsing_defs.h" */ -/* begin file simdjson/lsx/stringparsing_defs.h */ -#ifndef SIMDJSON_LSX_STRINGPARSING_DEFS_H -#define SIMDJSON_LSX_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + sizeof(v0)); - v0.store(dst); - v1.store(dst + sizeof(v0)); - - // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on LSX; therefore, we - // smash them together into a 64-byte mask and get the bitmask from there. - uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; -} - - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_LSX_STRINGPARSING_DEFS_H -/* end file simdjson/lsx/stringparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 -/* end file simdjson/lsx/begin.h */ -/* including simdjson/generic/amalgamated.h for lsx: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for lsx */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for lsx: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for lsx */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for lsx */ -/* including simdjson/generic/jsoncharutils.h for lsx: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for lsx */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for lsx */ -/* including simdjson/generic/atomparsing.h for lsx: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for lsx */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace lsx { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for lsx */ -/* including simdjson/generic/dom_parser_implementation.h for lsx: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for lsx */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace lsx -} // namespace simdjson - -namespace simdjson { -namespace lsx { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for lsx */ -/* including simdjson/generic/implementation_simdjson_result_base.h for lsx: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for lsx */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for lsx */ -/* including simdjson/generic/numberparsing.h for lsx: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for lsx */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace lsx { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for lsx */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for lsx: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for lsx */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for lsx */ -/* end file simdjson/generic/amalgamated.h for lsx */ -/* including simdjson/lsx/end.h: #include "simdjson/lsx/end.h" */ -/* begin file simdjson/lsx/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT -/* undefining SIMDJSON_IMPLEMENTATION from "lsx" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/lsx/end.h */ - -#endif // SIMDJSON_LSX_H -/* end file simdjson/lsx.h */ -/* including simdjson/lsx/implementation.h: #include */ -/* begin file simdjson/lsx/implementation.h */ -#ifndef SIMDJSON_LSX_IMPLEMENTATION_H -#define SIMDJSON_LSX_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation("lsx", "LoongArch SX", internal::instruction_set::LSX) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_LSX_IMPLEMENTATION_H -/* end file simdjson/lsx/implementation.h */ - -/* including simdjson/lsx/begin.h: #include */ -/* begin file simdjson/lsx/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "lsx" */ -#define SIMDJSON_IMPLEMENTATION lsx -/* including simdjson/lsx/base.h: #include "simdjson/lsx/base.h" */ -/* begin file simdjson/lsx/base.h */ -#ifndef SIMDJSON_LSX_BASE_H -#define SIMDJSON_LSX_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * Implementation for LSX. - */ -namespace lsx { - -class implementation; - -namespace { -namespace simd { -template struct simd8; -template struct simd8x64; -} // namespace simd -} // unnamed namespace - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_LSX_BASE_H -/* end file simdjson/lsx/base.h */ -/* including simdjson/lsx/intrinsics.h: #include "simdjson/lsx/intrinsics.h" */ -/* begin file simdjson/lsx/intrinsics.h */ -#ifndef SIMDJSON_LSX_INTRINSICS_H -#define SIMDJSON_LSX_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for LoongArch SX"); - -#endif // SIMDJSON_LSX_INTRINSICS_H -/* end file simdjson/lsx/intrinsics.h */ -/* including simdjson/lsx/bitmanipulation.h: #include "simdjson/lsx/bitmanipulation.h" */ -/* begin file simdjson/lsx/bitmanipulation.h */ -#ifndef SIMDJSON_LSX_BITMANIPULATION_H -#define SIMDJSON_LSX_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmask.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { - return __builtin_ctzll(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { - return __builtin_clzll(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int count_ones(uint64_t input_num) { - return __lsx_vpickve2gr_w(__lsx_vpcnt_d(__m128i(v2u64{input_num, 0})), 0); -} - - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_LSX_BITMANIPULATION_H -/* end file simdjson/lsx/bitmanipulation.h */ -/* including simdjson/lsx/bitmask.h: #include "simdjson/lsx/bitmask.h" */ -/* begin file simdjson/lsx/bitmask.h */ -#ifndef SIMDJSON_LSX_BITMASK_H -#define SIMDJSON_LSX_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; - return bitmask; -} - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif -/* end file simdjson/lsx/bitmask.h */ -/* including simdjson/lsx/numberparsing_defs.h: #include "simdjson/lsx/numberparsing_defs.h" */ -/* begin file simdjson/lsx/numberparsing_defs.h */ -#ifndef SIMDJSON_LSX_NUMBERPARSING_DEFS_H -#define SIMDJSON_LSX_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace lsx { -namespace numberparsing { - -// we don't have appropriate instructions, so let us use a scalar function -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - uint64_t val; - std::memcpy(&val, chars, sizeof(uint64_t)); - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); - return answer; -} - -} // namespace numberparsing -} // namespace lsx -} // namespace simdjson - -#ifndef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_IS_BIG_ENDIAN -#define SIMDJSON_SWAR_NUMBER_PARSING 0 -#else -#define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif -#endif - -#endif // SIMDJSON_LSX_NUMBERPARSING_DEFS_H -/* end file simdjson/lsx/numberparsing_defs.h */ -/* including simdjson/lsx/simd.h: #include "simdjson/lsx/simd.h" */ -/* begin file simdjson/lsx/simd.h */ -#ifndef SIMDJSON_LSX_SIMD_H -#define SIMDJSON_LSX_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{__m128i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i&() const { return this->value; } - simdjson_inline operator __m128i&() { return this->value; } - simdjson_inline operator const v16i8&() const { return (v16i8&)this->value; } - simdjson_inline operator v16i8&() { return (v16i8&)this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return __lsx_vor_v(*this, other); } - simdjson_inline Child operator&(const Child other) const { return __lsx_vand_v(*this, other); } - simdjson_inline Child operator^(const Child other) const { return __lsx_vxor_v(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return __lsx_vandn_v(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8: base> { - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return __lsx_vseq_b(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return __lsx_vor_v(__lsx_vbsll_v(*this, N), __lsx_vbsrl_v(prev_chunk, 16 - N)); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { - return __lsx_vreplgr2vr_b(uint8_t(-(!!_value))); - } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return __lsx_vpickve2gr_w(__lsx_vmskltz_b(*this), 0); } - simdjson_inline bool any() const { return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(*this), 0); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { return __lsx_vreplgr2vr_b(_value); } - static simdjson_inline simd8 zero() { return __lsx_vldi(0); } - static simdjson_inline simd8 load(const T values[16]) { - return __lsx_vld(reinterpret_cast(values), 0); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { - return __lsx_vst(*this, reinterpret_cast<__m128i *>(dst), 0); - } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return __lsx_vadd_b(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return __lsx_vsub_b(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return __lsx_vshuf_b(lookup_table, lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - template - simdjson_inline void compress(uint16_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by haswell - // lsx do it in 2 steps, first 8 bytes and then second 8 bytes... - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register. - __m128i shufmask = {int64_t(thintable_epi8[mask1]), int64_t(thintable_epi8[mask2]) + 0x0808080808080808}; - // this is the version "nearly pruned" - __m128i pruned = __lsx_vshuf_b(*this, *this, shufmask); - // we still need to put the pieces back together. - // we compute the popcount of the first words: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask - __m128i compactmask = __lsx_vldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop1 * 8); - __m128i answer = __lsx_vshuf_b(pruned, pruned, compactmask); - __lsx_vst(answer, reinterpret_cast(output), 0); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[16]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8({ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return __lsx_vmax_b(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return __lsx_vmin_b(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return __lsx_vslt_b(other, *this); } - simdjson_inline simd8 operator<(const simd8 other) const { return __lsx_vslt_b(*this, other); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(__m128i(v16u8{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - })) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return __lsx_vsadd_bu(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return __lsx_vssub_bu(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return __lsx_vmax_bu(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return __lsx_vmin_bu(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return 0 == __lsx_vpickve2gr_w(__lsx_vmskltz_b(*this), 0); } - simdjson_inline bool bits_not_set_anywhere() const { return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(*this), 0); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { - return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(__lsx_vand_v(*this, bits)), 0); - } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(__lsx_vsrli_b(*this, N)); } - template - simdjson_inline simd8 shl() const { return simd8(__lsx_vslli_b(*this, N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "LSX kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - uint16_t mask1 = uint16_t(mask); - uint16_t mask2 = uint16_t(mask >> 16); - uint16_t mask3 = uint16_t(mask >> 32); - uint16_t mask4 = uint16_t(mask >> 48); - __m128i zcnt = __lsx_vpcnt_h(__m128i(v2u64{~mask, 0})); - uint64_t zcnt1 = __lsx_vpickve2gr_hu(zcnt, 0); - uint64_t zcnt2 = __lsx_vpickve2gr_hu(zcnt, 1); - uint64_t zcnt3 = __lsx_vpickve2gr_hu(zcnt, 2); - uint64_t zcnt4 = __lsx_vpickve2gr_hu(zcnt, 3); - uint8_t *voutput = reinterpret_cast(output); - // There should be a critical value which processes in scaler is faster. - if (zcnt1) - this->chunks[0].compress(mask1, reinterpret_cast(voutput)); - voutput += zcnt1; - if (zcnt2) - this->chunks[1].compress(mask2, reinterpret_cast(voutput)); - voutput += zcnt2; - if (zcnt3) - this->chunks[2].compress(mask3, reinterpret_cast(voutput)); - voutput += zcnt3; - if (zcnt4) - this->chunks[3].compress(mask4, reinterpret_cast(voutput)); - voutput += zcnt4; - return reinterpret_cast(voutput) - reinterpret_cast(output); - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - this->chunks[2].store(ptr+sizeof(simd8)*2); - this->chunks[3].store(ptr+sizeof(simd8)*3); - } - - simdjson_inline uint64_t to_bitmask() const { - __m128i mask1 = __lsx_vmskltz_b(this->chunks[0]); - __m128i mask2 = __lsx_vmskltz_b(this->chunks[1]); - __m128i mask3 = __lsx_vmskltz_b(this->chunks[2]); - __m128i mask4 = __lsx_vmskltz_b(this->chunks[3]); - mask1 = __lsx_vilvl_h(mask2, mask1); - mask2 = __lsx_vilvl_h(mask4, mask3); - return __lsx_vpickve2gr_du(__lsx_vilvl_w(mask2, mask1), 0); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_LSX_SIMD_H -/* end file simdjson/lsx/simd.h */ -/* including simdjson/lsx/stringparsing_defs.h: #include "simdjson/lsx/stringparsing_defs.h" */ -/* begin file simdjson/lsx/stringparsing_defs.h */ -#ifndef SIMDJSON_LSX_STRINGPARSING_DEFS_H -#define SIMDJSON_LSX_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + sizeof(v0)); - v0.store(dst); - v1.store(dst + sizeof(v0)); - - // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on LSX; therefore, we - // smash them together into a 64-byte mask and get the bitmask from there. - uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; -} - - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_LSX_STRINGPARSING_DEFS_H -/* end file simdjson/lsx/stringparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 -/* end file simdjson/lsx/begin.h */ -/* including generic/amalgamated.h for lsx: #include */ -/* begin file generic/amalgamated.h for lsx */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) -#error generic/dependencies.h must be included before generic/amalgamated.h! -#endif - -/* including generic/base.h for lsx: #include */ -/* begin file generic/base.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { - -struct json_character_block; - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_BASE_H -/* end file generic/base.h for lsx */ -/* including generic/dom_parser_implementation.h for lsx: #include */ -/* begin file generic/dom_parser_implementation.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// Interface a dom parser implementation must fulfill -namespace simdjson { -namespace lsx { -namespace { - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); -simdjson_inline bool is_ascii(const simd8x64& input); - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file generic/dom_parser_implementation.h for lsx */ -/* including generic/json_character_block.h for lsx: #include */ -/* begin file generic/json_character_block.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { - -struct json_character_block { - static simdjson_inline json_character_block classify(const simd::simd8x64& in); - - simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_inline uint64_t op() const noexcept { return _op; } - simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } - - uint64_t _whitespace; - uint64_t _op; -}; - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H -/* end file generic/json_character_block.h for lsx */ -/* end file generic/amalgamated.h for lsx */ -/* including generic/stage1/amalgamated.h for lsx: #include */ -/* begin file generic/stage1/amalgamated.h for lsx */ -// Stuff other things depend on -/* including generic/stage1/base.h for lsx: #include */ -/* begin file generic/stage1/base.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace stage1 { - -class bit_indexer; -template -struct buf_block_reader; -struct json_block; -class json_minifier; -class json_scanner; -struct json_string_block; -class json_string_scanner; -class json_structural_indexer; - -} // namespace stage1 - -namespace utf8_validation { -struct utf8_checker; -} // namespace utf8_validation - -using utf8_validation::utf8_checker; - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H -/* end file generic/stage1/base.h for lsx */ -/* including generic/stage1/buf_block_reader.h for lsx: #include */ -/* begin file generic/stage1/buf_block_reader.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace lsx { -namespace { -namespace stage1 { - -// Walks through a buffer in block-sized increments, loading the last part with spaces -template -struct buf_block_reader { -public: - simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t *full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t *dst) const; - /** - * Like get_remainder, for input followed by SIMDJSON_PADDING readable bytes. - * - * get_remainder writes the block with a memset and a memcpy of the remaining - * length, and the vector loads that read it back cannot be forwarded from - * those narrower stores: they wait until the stores reach the cache, which - * costs tens of cycles on every document. Here the block is put together in - * vector registers, from loads that may read into the padding, and written - * with full-width stores that the loads can forward from. - */ - simdjson_inline size_t get_remainder_padded(uint8_t *dst) const; - simdjson_inline void advance(); -private: - const uint8_t *buf; - const size_t len; - const size_t lenminusstep; - size_t idx; -}; - -template -simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} - -template -simdjson_inline size_t buf_block_reader::block_index() { return idx; } - -template -simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; -} - -template -simdjson_inline const uint8_t *buf_block_reader::full_block() const { - return &buf[idx]; -} - -template -simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { - if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; -} - -// keep_mask[128 - n + i] is 0xFF for i < n and 0 otherwise. -alignas(64) static const uint8_t keep_mask[256] = { - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF -}; - -template -simdjson_inline size_t buf_block_reader::get_remainder_padded(uint8_t *dst) const { - static_assert(STEP_SIZE <= 128, "keep_mask covers blocks of up to 128 bytes"); - static_assert(sizeof(simd::simd8) <= SIMDJSON_PADDING, "a vector load at the last byte must stay in the padding"); - const size_t remaining = len - idx; - if (remaining == 0) { return 0; } - const simd::simd8 spaces = simd::simd8::splat(0x20); - for (size_t off = 0; off < STEP_SIZE; off += sizeof(simd::simd8)) { - simd::simd8 v = spaces; - if (off < remaining) { - // Starts before len, so it ends within the padding. - const simd::simd8 in = simd::simd8::load(buf + idx + off); - const simd::simd8 keep = simd::simd8::load(keep_mask + 128 - remaining + off); - v = spaces ^ ((in ^ spaces) & keep); - } - v.store(dst + off); - } - return remaining; -} - -template -simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for lsx */ -/* including generic/stage1/json_escape_scanner.h for lsx: #include */ -/* begin file generic/stage1/json_escape_scanner.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace stage1 { - -/** - * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). - */ -struct json_escape_scanner { - /** The actual escape characters (the backslashes themselves). */ - uint64_t next_is_escaped = 0ULL; - - struct escaped_and_escape { - /** - * Mask of escaped characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 0100100010100101000 - * n \ \ n \ \ - * ``` - */ - uint64_t escaped; - /** - * Mask of escape characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 1001000101001010001 - * \ \ \ \ \ \ \ - * ``` - */ - uint64_t escape; - }; - - /** - * Get a mask of both escape and escaped characters (the characters following a backslash). - * - * @param potential_escape A mask of the character that can escape others (but could be - * escaped itself). e.g. block.eq('\\') - */ - simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { - -#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT - if (!backslash) { return {next_escaped_without_backslashes(), 0}; } -#endif - - // | | Mask (shows characters instead of 1's) | Depth | Instructions | - // |--------------------------------|----------------------------------------|-------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | - // | | ` even odd even odd odd` | | | - // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) - // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) - // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) - // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () - // (*) this is not needed until the next iteration - uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); - uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); - uint64_t escape = escape_and_terminal_code & backslash; - this->next_is_escaped = escape >> 63; - return {escaped, escape}; - } - -private: - static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; - - simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { - uint64_t escaped = this->next_is_escaped; - this->next_is_escaped = 0; - return escaped; - } - - /** - * Returns a mask of the next escape characters (masking out escaped backslashes), along with - * any non-backslash escape codes. - * - * \n \\n \\\n \\\\n returns: - * \n \ \ \n \ \ - * 11 100 1011 10100 - * - * You are expected to mask out the first bit yourself if the previous block had a trailing - * escape. - * - * & the result with potential_escape to get just the escape characters. - * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. - */ - static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { - // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: - // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be - // inverted (\\\ would be 010 instead of 101). - // - // ``` - // string: | ____\\\\_\\\\_____ | - // maybe_escaped | ODD | \ \ \ \ | - // even-aligned ^^^ ^^^^ odd-aligned - // ``` - // - // Taking that into account, our basic strategy is: - // - // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for - // odd-aligned runs. - // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the - // odd bits in odd-aligned runs. - // 3. & with backslash to clean up any stray bits. - // runs are set to 0, and then XORing with "odd": - // - // | | Mask (shows characters instead of 1's) | Instructions | - // |--------------------------------|----------------------------------------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | - // | | ` even odd even odd odd` | - // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) - // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) - // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) - // - - // Escaped characters are characters following an escape. - uint64_t maybe_escaped = potential_escape << 1; - - // To distinguish odd from even escape sequences, therefore, we turn on any *starting* - // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) - // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. - // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. - // - All other odd bytes are 1, and even bytes are 0. - uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; - uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; - - // Now we flip all odd bytes back with xor. This: - // - Makes odd runs of backslashes go from 0000 to 1010 - // - Makes even runs of backslashes go from 1111 to 1010 - // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) - // - Resets all other bytes to 0 - return even_series_codes_and_odd_bits ^ ODD_BITS; - } -}; - -} // namespace stage1 -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_escape_scanner.h for lsx */ -/* including generic/stage1/json_string_scanner.h for lsx: #include */ -/* begin file generic/stage1/json_string_scanner.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace stage1 { - -struct json_string_block { - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : - _escaped(escaped), _quote(quote), _in_string(in_string) {} - - // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } - // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } - // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } - // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } - - // escaped characters (backslashed--does not include the hex characters after \u) - uint64_t _escaped; - // real quotes (non-escaped ones) - uint64_t _quote; - // string characters (includes start quote but not end quote) - uint64_t _in_string; -}; - -// Scans blocks for string characters, storing the state necessary to do so -class json_string_scanner { -public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); - -private: - // Scans for escape characters - json_escape_scanner escape_scanner{}; - // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). - uint64_t prev_in_string = 0ULL; -}; - -// -// Return a mask of all string characters plus end quotes. -// -// prev_escaped is overflow saying whether the next character is escaped. -// prev_in_string is overflow saying whether we're still in a string. -// -// Backslash sequences outside of quotes will be detected in stage 2. -// -simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = escape_scanner.next(backslash).escaped; - const uint64_t quote = in.eq('"') & ~escaped; - - // - // prefix_xor flips on bits inside the string (and flips off the end quote). - // - // Then we xor with prev_in_string: if we were in a string already, its effect is flipped - // (characters inside strings are outside, and characters outside strings are inside). - // - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - - // - // Check if we're still in a string at the end of the box so the next block will know - // - prev_in_string = uint64_t(static_cast(in_string) >> 63); - - // Use ^ to turn the beginning quote off, and the end quote on. - - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_string_block(escaped, quote, in_string); -} - -simdjson_really_inline error_code json_string_scanner::finish() { - if (prev_in_string) { - return UNCLOSED_STRING; - } - return SUCCESS; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_string_scanner.h for lsx */ -/* including generic/stage1/utf8_lookup4_algorithm.h for lsx: #include */ -/* begin file generic/stage1/utf8_lookup4_algorithm.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace utf8_validation { - -using namespace simd; - - simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { -// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) -// Bit 1 = Too Long (ASCII followed by continuation) -// Bit 2 = Overlong 3-byte -// Bit 4 = Surrogate -// Bit 5 = Overlong 2-byte -// Bit 7 = Two Continuations - constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ - // 11______ 11______ - constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ - constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ - constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ - constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ - constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ - constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ - // 11110100 101_____ - // 11110101 1001____ - // 11110101 101_____ - // 1111011_ 1001____ - // 1111011_ 101_____ - // 11111___ 1001____ - // 11111___ 101_____ - constexpr const uint8_t TOO_LARGE_1000 = 1<<6; - // 11110101 1000____ - // 1111011_ 1000____ - // 11111___ 1000____ - constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ - - const simd8 byte_1_high = prev1.shr<4>().lookup_16( - // 0_______ ________ - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - // 10______ ________ - TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, - // 1100____ ________ - TOO_SHORT | OVERLONG_2, - // 1101____ ________ - TOO_SHORT, - // 1110____ ________ - TOO_SHORT | OVERLONG_3 | SURROGATE, - // 1111____ ________ - TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 - ); - constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( - // ____0000 ________ - CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, - // ____0001 ________ - CARRY | OVERLONG_2, - // ____001_ ________ - CARRY, - CARRY, - - // ____0100 ________ - CARRY | TOO_LARGE, - // ____0101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____011_ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - - // ____1___ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____1101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000 - ); - const simd8 byte_2_high = input.shr<4>().lookup_16( - // ________ 0_______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - - // ________ 1000____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, - // ________ 1001____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, - // ________ 101_____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - - // ________ 11______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT - ); - return (byte_1_high & byte_1_low & byte_2_high); - } - simdjson_inline simd8 check_multibyte_lengths(const simd8 input, - const simd8 prev_input, const simd8 sc) { - simd8 prev2 = input.prev<2>(prev_input); - simd8 prev3 = input.prev<3>(prev_input); - simd8 must23 = must_be_2_3_continuation(prev2, prev3); - simd8 must23_80 = must23 & uint8_t(0x80); - return must23_80 ^ sc; - } - - // - // Return nonzero if there are incomplete multibyte characters at the end of the block: - // e.g. if there is a 4-byte character, but it's 3 bytes from the end. - // - simdjson_inline simd8 is_incomplete(const simd8 input) { - // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): - // ... 1111____ 111_____ 11______ -#if SIMDJSON_IMPLEMENTATION_ICELAKE || (SIMDJSON_IMPLEMENTATION_RVV_VLS && __riscv_v_fixed_vlen >= 512) - static const uint8_t max_array[64] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#else - static const uint8_t max_array[32] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#endif - const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); - return input.gt_bits(max_value); - } - - struct utf8_checker { - // If this is nonzero, there has been a UTF-8 error. - simd8 error; - // The last input we received - simd8 prev_input_block; - // Whether the last input we received was incomplete (used for ASCII fast path) - simd8 prev_incomplete; - - // - // Check whether the current bytes are valid UTF-8. - // - simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { - // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes - // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) - simd8 prev1 = input.prev<1>(prev_input); - simd8 sc = check_special_cases(input, prev1); - this->error |= check_multibyte_lengths(input, prev_input, sc); - } - - // The only problem that can happen at EOF is that a multibyte character is too short - // or a byte value too large in the last bytes: check_special_cases only checks for bytes - // too large in the first of two bytes. - simdjson_inline void check_eof() { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; - } - - simdjson_inline void check_next_input(const simd8x64& input) { - if(simdjson_likely(is_ascii(input))) { - this->error |= this->prev_incomplete; - } else { - // you might think that a for-loop would work, but under Visual Studio, it is not good enough. - static_assert((simd8x64::NUM_CHUNKS == 1) - ||(simd8x64::NUM_CHUNKS == 2) - || (simd8x64::NUM_CHUNKS == 4), - "We support one, two or four chunks per 64-byte block."); - SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 1) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 2) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 4) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - this->check_utf8_bytes(input.get<2>(), input.get<1>()); - this->check_utf8_bytes(input.get<3>(), input.get<2>()); - } - this->prev_incomplete = is_incomplete(input.get::NUM_CHUNKS-1>()); - this->prev_input_block = input.get::NUM_CHUNKS-1>(); - } - } - // do not forget to call check_eof! - simdjson_warn_unused simdjson_inline error_code errors() { - return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; - } - - }; // struct utf8_checker -} // namespace utf8_validation - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H -/* end file generic/stage1/utf8_lookup4_algorithm.h for lsx */ -/* including generic/stage1/json_scanner.h for lsx: #include */ -/* begin file generic/stage1/json_scanner.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace stage1 { - -/** - * A block of scanned json, with information on operators and scalars. - * - * We seek to identify pseudo-structural characters. Anything that is inside - * a string must be omitted (hence & ~_string.string_tail()). - * Otherwise, pseudo-structural characters come in two forms. - * 1. We have the structural characters ([,],{,},:, comma). The - * term 'structural character' is from the JSON RFC. - * 2. We have the 'scalar pseudo-structural characters'. - * Scalars are quotes, and any character except structural characters and white space. - * - * To identify the scalar pseudo-structural characters, we must look at what comes - * before them: it must be a space, a quote or a structural characters. - * Starting with simdjson v0.3, we identify them by - * negation: we identify everything that is followed by a non-quote scalar, - * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. - */ -struct json_block { -public: - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - - /** - * The start of structurals. - * In simdjson prior to v0.3, these were called the pseudo-structural characters. - **/ - simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } - /** All JSON whitespace (i.e. not in a string) */ - simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } - - // Helpers - - /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } - /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } - - // string and escape characters - json_string_block _string; - // whitespace, structural characters ('operators'), scalars - json_character_block _characters; - // whether the previous character was a scalar - uint64_t _follows_potential_nonquote_scalar; -private: - // Potential structurals (i.e. disregarding strings) - - /** - * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". - * They may reside inside a string. - **/ - simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } - /** - * The start of non-operator runs, like 123, true and "abc". - * It main reside inside a string. - **/ - simdjson_inline uint64_t potential_scalar_start() const noexcept { - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space - // then we know that it is irrelevant structurally. - return _characters.scalar() & ~follows_potential_scalar(); - } - /** - * Whether the given character is immediately after a non-operator like 123, true. - * The characters following a quote are not included. - */ - simdjson_inline uint64_t follows_potential_scalar() const noexcept { - // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character - // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a - // white space. - // It is understood that within quoted region, anything at all could be marked (irrelevant). - return _follows_potential_nonquote_scalar; - } -}; - -/** - * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. - * - * The scanner starts by calculating two distinct things: - * - string characters (taking \" into account) - * - structural characters or 'operators' ([]{},:, comma) - * and scalars (runs of non-operators like 123, true and "abc") - * - * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: - * in particular, the operator/scalar bit will find plenty of things that are actually part of - * strings. When we're done, json_block will fuse the two together by masking out tokens that are - * part of a string. - */ -class json_scanner { -public: - json_scanner() = default; - simdjson_inline json_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_warn_unused simdjson_inline error_code finish(); - -private: - // Whether the last character of the previous iteration is part of a scalar token - // (anything except whitespace or a structural character/'operator'). - uint64_t prev_scalar = 0ULL; - json_string_scanner string_scanner{}; -}; - - -// -// Check if the current character immediately follows a matching character. -// -// For example, this checks for quotes with backslashes in front of them: -// -// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); -// -simdjson_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; -} - -simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { - json_string_block strings = string_scanner.next(in); - // identifies the white-space and the structural characters - json_character_block characters = json_character_block::classify(in); - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). - // - // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) - // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential - // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we - // may need to add an extra check when parsing strings. - // - // Performance: there are many ways to skin this cat. - const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); - uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_block( - strings,// strings is a function-local object so either it moves or the copy is elided. - characters, - follows_nonquote_scalar - ); -} - -simdjson_warn_unused simdjson_inline error_code json_scanner::finish() { - return string_scanner.finish(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H -/* end file generic/stage1/json_scanner.h for lsx */ - -// All other declarations -/* including generic/stage1/find_next_document_index.h for lsx: #include */ -/* begin file generic/stage1/find_next_document_index.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace stage1 { - -/** - * This algorithm is used to quickly identify the last structural position that - * makes up a complete document. - * - * It does this by going backwards and finding the last *document boundary* (a - * place where one value follows another without a comma between them). If the - * last document (the characters after the boundary) has an equal number of - * start and end brackets, it is considered complete. - * - * Simply put, we iterate over the structural characters, starting from - * the end. We consider that we found the end of a JSON document when the - * first element of the pair is NOT one of these characters: '{' '[' ':' ',' - * and when the second element is NOT one of these characters: '}' ']' ':' ','. - * - * This simple comparison works most of the time, but it does not cover cases - * where the batch's structural indexes contain a perfect amount of documents. - * In such a case, we do not have access to the structural index which follows - * the last document, therefore, we do not have access to the second element in - * the pair, and that means we cannot identify the last document. To fix this - * issue, we keep a count of the open and closed curly/square braces we found - * while searching for the pair. When we find a pair AND the count of open and - * closed curly/square braces is the same, we know that we just passed a - * complete document, therefore the last json buffer location is the end of the - * batch. - */ -simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { - const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - const uint8_t e = parser.buf[len - 1]; - return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && - e != ' ' && e != '\t' && e != '\n' && e != '\r'; -} - -/** - * The structural index past the last one of a partial batch is the end of this - * window, not a token of it: the byte there is the first byte of the next - * window. A finished value must not be rejected because that byte is ',', ':', - * ']' or '}'. A last document that itself ends with '{', '[', ',' or ':' is - * invalid JSON, yet its brackets may balance: walking it, the On-Demand API - * could read the next batch as a value. We leave such a document to the next - * batch, so that the batch ends between two documents. - */ -simdjson_inline bool ends_with_unfinished_document(dom_parser_implementation &parser) { - const uint8_t last = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - return last == '{' || last == '[' || last == ',' || last == ':'; -} - -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { - // Variant: do not count separately, just figure out depth - if(parser.n_structural_indexes == 0) { return 0; } - auto arr_cnt = 0; - auto obj_cnt = 0; - for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { - auto idxb = parser.structural_indexes[i]; - switch (parser.buf[idxb]) { - case ':': - case ',': - continue; - case '}': - obj_cnt--; - continue; - case ']': - arr_cnt--; - continue; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - auto idxa = parser.structural_indexes[i - 1]; - switch (parser.buf[idxa]) { - case '{': - case '[': - case ':': - case ',': - continue; - } - // Last document is complete, so the next document will appear after! - if (!arr_cnt && !obj_cnt) { - return defer_last ? i : parser.n_structural_indexes; - } - // Last document is incomplete; mark the document at i + 1 as the next one - return i; - } - // If we made it to the end, we want to finish counting to see if we have a full document. - switch (parser.buf[parser.structural_indexes[0]]) { - case '}': - obj_cnt--; - break; - case ']': - arr_cnt--; - break; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - if (!arr_cnt && !obj_cnt) { - // We have a complete document. - return defer_last ? 0 : parser.n_structural_indexes; - } - return 0; -} - -/** - * Sentinel value returned to indicate a document started but didn't fit - * (CAPACITY error), as opposed to 0 which means no document content found - * (EMPTY). - */ -constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; - -/** - * For RFC 7464 JSON text sequences, filter RS from structural indexes and - * find batch boundaries. - * - * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. - * RS bytes appear in structural_indexes as they are classified as scalars. - * This function: - * 1. Scans structural_indexes to find and count RS positions - * 2. Filters RS out of structural_indexes in-place - * 3. Determines batch boundaries based on RS positions - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @param scan_len Offset past which no value start may be derived. Equals len - * unless stage 1 dropped a trailing unclosed string, whose bytes it - * never validated. - * @return The number of structural indexes to keep (after RS filtering), - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t find_next_document_index_json_sequence( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start, - size_t scan_len) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Phase 1: Scan structural_indexes to find RS positions and handle them. - // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS - // is already in structural_indexes (it's an operator). For scalars like numbers, - // the digit following RS is NOT in structural_indexes because the scanner sees - // RS as a scalar, making the digit a scalar continuation, not a start. - // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the - // actual value start position. - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_rs_pos = 0; - uint32_t rs_count = 0; - - for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { - const uint32_t pos = parser.structural_indexes[read_idx]; - if (parser.buf[pos] == 0x1E) { - // This is an RS character - find the actual JSON value start. - last_rs_pos = pos; - rs_count++; - // Skip past this RS and any whitespace *and any additional RSes* - // to locate the real value. Consecutive RSes are degenerate - // "empty records" per RFC 7464; we collapse them here. They do - // not always appear as separate entries in structural_indexes - // because the scanner groups runs of adjacent non-whitespace - // scalar bytes (including RS) into a single scalar start. - uint32_t value_start = pos + 1; - while (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { - value_start++; - } else if (c == 0x1E) { - // Collapsed empty record. Still count it so rs_count reflects - // the true number of record markers and last_rs_pos tracks - // the final one. - last_rs_pos = value_start; - rs_count++; - value_start++; - } else { - break; - } - } - // If the scanner emitted additional structurals inside the - // whitespace+RS run we just walked over (i.e., isolated RSes - // separated by whitespace), skip past them so we do not - // double-count or double-emit. - while (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] < value_start) { - read_idx++; - } - // Check if the value start is an operator (always present in - // scanner structural_indexes) or a scalar-like start (which may - // be missing from structural_indexes and must be added here). - // Note: '"' is NOT always in structural_indexes. The scanner - // classifies '"' as a scalar character and emits it as a - // structural only when it is a *scalar start* (preceded by - // whitespace or an operator). When '"' immediately follows an - // RS (which the scanner also classifies as scalar), it is - // treated as a scalar continuation and not emitted - so we - // must add it here just like any other scalar value. - if (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - const bool is_operator = - (c == '{' || c == '}' || c == '[' || c == ']' || - c == ':' || c == ','); - // If the next scanner structural is exactly at value_start, - // the scanner already emitted it (it followed whitespace) and - // we must not add a duplicate - a subsequent iteration will - // copy it into write_idx. - const bool already_emitted = - (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] == value_start); - if (!is_operator && !already_emitted) { - // Scalar value (number/true/false/null/string) - add its - // position since scanner missed it. - parser.structural_indexes[write_idx++] = value_start; - } - } - } else { - // Not RS, copy to output - parser.structural_indexes[write_idx++] = pos; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { - // Only RS markers here: the last one opens a record continuing past the - // window, so that is where the next batch resumes. - if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } - return 0; - } - if (rs_count == 0) { - // No RS found; for final batch, try generic boundary detection - return is_final ? find_next_document_index(parser) : 0; - } - - // Phase 2: Determine batch boundaries based on RS positions - - if (is_final) { - // Final batch: all documents are complete (last one ends at EOF). - // In json_sequence mode, RS markers define document boundaries, so all - // remaining structurals form complete documents. Return them all directly. - // (Calling find_next_document_index() would fail for scalar documents.) - return parser.n_structural_indexes; - } - - // Partial batch: need to find complete documents only. - // A document starting at an RS is complete if there is another RS after it. - next_batch_start = last_rs_pos; - - if (rs_count < 2) { - // Only one RS, so we have at most one document that may be incomplete. - // We cannot confirm it is complete without another RS. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least 2 RS markers. The last complete document ends before last_rs_pos. - - // Find the structural index cutoff: keep only structurals < last_rs_pos. - // Since we already filtered RS, all remaining structurals are valid. - // We iterate backward to find the last structural before last_rs_pos. - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_rs_pos) { - keep_count = i; - break; - } - } - - // No structurals before the last RS - no complete documents - if (keep_count == 0) { return 0; } - - // All documents before the last RS are complete by definition (the next RS - // confirms their end). No need to call find_next_document_index() which - // would fail for scalar documents like `1` or `"hello"`. - return keep_count; -} - -/** - * Filter comma-delimited documents by removing root-level commas from - * structural indexes. - * - * For comma-delimited format like `{...},{...},{...}`, we need to remove - * the commas that separate documents (depth 0) while preserving commas - * inside arrays and objects (depth > 0). - * - * After filtering, the structural indexes look like whitespace-delimited - * documents, so find_next_document_index() works unchanged. - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @return The number of structural indexes to keep, - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t filter_comma_delimited( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Track depth to identify root-level commas (depth 0) - int depth = 0; - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_root_comma_pos = 0; - uint32_t root_comma_count = 0; - - for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { - uint32_t idx = parser.structural_indexes[i]; - uint8_t c = parser.buf[idx]; - - switch (c) { - case '{': case '[': - depth++; - parser.structural_indexes[write_idx++] = idx; - break; - case '}': case ']': - depth--; - parser.structural_indexes[write_idx++] = idx; - break; - case ',': - if (depth == 0) { - // Root-level comma = document boundary, skip it - last_root_comma_pos = idx; - root_comma_count++; - continue; - } - parser.structural_indexes[write_idx++] = idx; - break; - default: - // Colons, scalars, etc. - parser.structural_indexes[write_idx++] = idx; - break; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { return 0; } - - if (is_final) { - // Final batch: use standard boundary detection on filtered indexes - return find_next_document_index(parser); - } - - // Partial batch: need to find complete documents only. - // A document ending with a root comma is complete. - if (root_comma_count == 0) { - // No root commas found; we cannot confirm any document is complete. - // The whole batch might be one incomplete document. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least one root comma. Documents before the last comma are complete. - next_batch_start = last_root_comma_pos + 1; - - // Find the structural index cutoff: keep only structurals < last_root_comma_pos - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_root_comma_pos) { - keep_count = i; - break; - } - } - - if (keep_count == 0) { return 0; } - - // Use standard boundary detection on the complete portion - parser.n_structural_indexes = keep_count; - return find_next_document_index(parser); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H -/* end file generic/stage1/find_next_document_index.h for lsx */ -/* including generic/stage1/json_minifier.h for lsx: #include */ -/* begin file generic/stage1/json_minifier.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace lsx { -namespace { -namespace stage1 { - -class json_minifier { -public: - template - static error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept; - -private: - simdjson_inline json_minifier(uint8_t *_dst) - : dst{_dst} - {} - template - simdjson_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_warn_unused simdjson_inline error_code finish(uint8_t *dst_start, size_t &dst_len); - json_scanner scanner{}; - uint8_t *dst; -}; - -simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { - uint64_t mask = block.whitespace(); - dst += in.compress(mask, dst); -} - -simdjson_warn_unused simdjson_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { - error_code error = scanner.finish(); - if (error) { dst_len = 0; return error; } - dst_len = dst - dst_start; - return SUCCESS; -} - -template<> -simdjson_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - simd::simd8x64 in_2(block_buf+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1); - this->next(in_2, block_2); - reader.advance(); -} - -template<> -simdjson_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - json_block block_1 = scanner.next(in_1); - this->next(block_buf, block_1); - reader.advance(); -} - -template -error_code json_minifier::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept { - buf_block_reader reader(buf, len); - json_minifier minifier(dst); - - // Index the first n-1 blocks - while (reader.has_full_block()) { - minifier.step(reader.full_block(), reader); - } - - // Index the last (remainder) block, padded with spaces - uint8_t block[STEP_SIZE]; - size_t remaining_bytes = reader.get_remainder(block); - if (remaining_bytes > 0) { - // We do not want to write directly to the output stream. Rather, we write - // to a local buffer (for safety). - uint8_t out_block[STEP_SIZE]; - uint8_t * const guarded_dst{minifier.dst}; - minifier.dst = out_block; - minifier.step(block, reader); - size_t to_write = minifier.dst - out_block; - // In some cases, we could be enticed to consider the padded spaces - // as part of the string. This is fine as long as we do not write more - // than we consumed. - if(to_write > remaining_bytes) { to_write = remaining_bytes; } - memcpy(guarded_dst, out_block, to_write); - minifier.dst = guarded_dst + to_write; - } - return minifier.finish(dst, dst_len); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H -/* end file generic/stage1/json_minifier.h for lsx */ -/* including generic/stage1/json_structural_indexer.h for lsx: #include */ -/* begin file generic/stage1/json_structural_indexer.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace lsx { -namespace { -namespace stage1 { - -class bit_indexer { -public: - uint32_t *tail; - - simdjson_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} - -#if SIMDJSON_PREFER_REVERSE_BITS - /** - * ARM lacks a fast trailing zero instruction, but it has a fast - * bit reversal instruction and a fast leading zero instruction. - * Thus it may be profitable to reverse the bits (once) and then - * to rely on a sequence of instructions that call the leading - * zero instruction. - * - * Performance notes: - * The chosen routine is not optimal in terms of data dependency - * since zero_leading_bit might require two instructions. However, - * it tends to minimize the total number of instructions which is - * beneficial. - */ - simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { - int lz = leading_zeroes(rev_bits); - this->tail[i] = static_cast(idx) + lz; - rev_bits = zero_leading_bit(rev_bits, lz); - } -#else - /** - * Under recent x64 systems, we often have both a fast trailing zero - * instruction and a fast 'clear-lower-bit' instruction so the following - * algorithm can be competitive. - */ - - simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { - this->tail[i] = idx + trailing_zeroes(bits); - bits = clear_lowest_bit(bits); - } -#endif // SIMDJSON_PREFER_REVERSE_BITS - - template - simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { - write_index(idx, bits, START); - SIMDJSON_IF_CONSTEXPR (N > 1) { - write_indexes<(N-1>0?START+1:START), (N-1>=0?N-1:1)>(idx, bits); - } - return START+N; - } - - template - simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { - write_indexes(idx, bits); - SIMDJSON_IF_CONSTEXPR ((START+STEP) < END) { - if (simdjson_unlikely((START+STEP) < cnt)) { - write_indexes_stepped<(START+STEP(idx, bits, cnt); - } - } - return ((END-START) % STEP) == 0 ? END : (END-START) - ((END-START) % STEP) + STEP; - } - - // flatten out values in 'bits' assuming that they are are to have values of idx - // plus their position in the bitvector, and store these indexes at - // base_ptr[base] incrementing base as we go - // will potentially store extra values beyond end of valid bits, so base_ptr - // needs to be large enough to handle this - // - // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it - // will provide its own version of the code. -#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - simdjson_inline void write(uint32_t idx, uint64_t bits); -#else - simdjson_inline void write(uint32_t idx, uint64_t bits) { - // In some instances, the next branch is expensive because it is mispredicted. - // Unfortunately, in other cases, - // it helps tremendously. - if (bits == 0) - return; - - int cnt = static_cast(count_ones(bits)); - -#if SIMDJSON_PREFER_REVERSE_BITS - bits = reverse_bits(bits); -#endif -#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP - static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; -#else - static constexpr const int STEP = 4; -#endif - static constexpr const int STEP_UNTIL = 24; - - write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); - SIMDJSON_IF_CONSTEXPR (STEP_UNTIL < 64) { - if (simdjson_unlikely(STEP_UNTIL < cnt)) { - for (int i=STEP_UNTIL; itail += cnt; - } -#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -}; - -class json_structural_indexer { -public: - /** - * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. - * - * @param partial Setting the partial parameter to true allows the find_structural_bits to - * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If - * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. - */ - template - static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept; - -private: - simdjson_inline json_structural_indexer(uint32_t *structural_indexes); - template - simdjson_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_warn_unused simdjson_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); - - json_scanner scanner{}; - utf8_checker checker{}; - bit_indexer indexer; - uint64_t prev_structurals = 0; - uint64_t unescaped_chars_error = 0; -}; - -simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} - -// Skip the last character if it is partial -simdjson_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { - if (simdjson_unlikely(len < 3)) { - switch (len) { - case 2: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left - return len; - case 1: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - return len; - case 0: - return len; - } - } - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left - return len; -} - -// -// PERF NOTES: -// We pipe 2 inputs through these stages: -// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load -// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. -// 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. -// The output of step 1 depends entirely on this information. These functions don't quite use -// up enough CPU: the second half of the functions is highly serial, only using 1 execution core -// at a time. The second input's scans has some dependency on the first ones finishing it, but -// they can make a lot of progress before they need that information. -// 3. Step 1 does not use enough capacity, so we run some extra stuff while we're waiting for that -// to finish: utf-8 checks and generating the output from the last iteration. -// -// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all -// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough -// workout. -// -template -error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept { - if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } - // We guard the rest of the code so that we can assume that len > 0 throughout. - if (len == 0) { return EMPTY; } - if (is_streaming(partial)) { - len = trim_partial_utf8(buf, len); - // If you end up with an empty window after trimming - // the partial UTF-8 bytes, then chances are good that you - // have an UTF-8 formatting error. - if(len == 0) { return UTF8_ERROR; } - } - buf_block_reader reader(buf, len); - json_structural_indexer indexer(parser.structural_indexes.get()); - - // Read all but the last block - while (reader.has_full_block()) { - indexer.step(reader.full_block(), reader); - } - // Take care of the last block (will always be there unless file is empty which is - // not supposed to happen.) - uint8_t block[STEP_SIZE]; - const size_t remaining = parser._unpadded ? reader.get_remainder(block) : reader.get_remainder_padded(block); - if (simdjson_unlikely(remaining == 0)) { return UNEXPECTED_ERROR; } - indexer.step(block, reader); - return indexer.finish(parser, reader.block_index(), len, partial); -} - -template<> -simdjson_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block); - simd::simd8x64 in_2(block+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1, reader.block_index()); - this->next(in_2, block_2, reader.block_index()+64); - reader.advance(); -} - -template<> -simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block); - json_block block_1 = scanner.next(in_1); - this->next(in_1, block_1, reader.block_index()); - reader.advance(); -} - -simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { - uint64_t unescaped = in.lteq(0x1F); -#if SIMDJSON_UTF8VALIDATION - checker.check_next_input(in); -#endif - indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser - prev_structurals = block.structural_start(); - unescaped_chars_error |= block.non_quote_inside_string(unescaped); -} - -// Drops the unclosed string that ends a json_sequence batch and leaves its -// start in structural_indexes[n_structural_indexes]. The last index is its -// opening quote, unless the quote directly follows an RS: the scanner reads -// RS as a scalar byte, so it indexes the RS and not the quote. Keep the RS. -simdjson_inline void drop_json_sequence_unclosed_string(dom_parser_implementation &parser, size_t len) noexcept { - uint32_t quote = parser.structural_indexes[parser.n_structural_indexes - 1]; - if (parser.buf[quote] == '"') { - parser.n_structural_indexes--; - return; - } - while (quote < len && parser.buf[quote] != '"') { quote++; } - parser.structural_indexes[parser.n_structural_indexes] = quote; -} - -simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial) { - // Write out the final iteration's structurals - indexer.write(uint32_t(idx-64), prev_structurals); - error_code error = scanner.finish(); - // We deliberately break down the next expression so that it is - // human readable. - const bool should_we_exit = is_streaming(partial) ? - ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING - : (error != SUCCESS); // if partial is false, we must have SUCCESS - const bool have_unclosed_string = (error == UNCLOSED_STRING); - parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); - /*** - * The On-Demand API requires special padding. - * - * This is related to https://github.com/simdjson/simdjson/issues/906 - * Basically, we want to make sure that if the parsing continues beyond the last (valid) - * structural character, it quickly stops. - * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. - * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing - * continues, then it must be [,] or }. - * Suppose it is ] or }. We backtrack to the first character, what could it be that would - * not trigger an error? It could be ] or } but no, because you can't start a document that way. - * It can't be a comma, a colon or any simple value. So the only way we could continue is - * if the repeated character is [. But if so, the document must start with [. But if the document - * starts with [, it should end with ]. If we enforce that rule, then we would get - * ][[ which is invalid. - * - * This is illustrated with the test array_iterate_unclosed_error() on the following input: - * R"({ "a": [,,)" - **/ - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final - parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); - parser.structural_indexes[parser.n_structural_indexes + 2] = 0; - parser.next_structural_index = 0; - - // Bail out only once the count and sentinels above are set: - // document_stream::truncated_bytes() reads them even on error. - if (simdjson_unlikely(should_we_exit)) { return error; } - if (unescaped_chars_error) { return UNESCAPED_CHARS; } - - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - return EMPTY; - } - if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { - return UNEXPECTED_ERROR; - } - if (partial == stage1_mode::streaming_partial) { - // If we have an unclosed string, then the last structural - // will be the quote and we want to make sure to omit it. - if(have_unclosed_string) { - parser.n_structural_indexes--; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - // We truncate the input to the end of the last complete document (or zero). - const bool partial_scalar = !have_unclosed_string && ends_with_partial_scalar(parser, len); - const bool unfinished = ends_with_unfinished_document(parser); - auto new_structural_indexes = find_next_document_index(parser, partial_scalar || unfinished); - if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { - // The batch holds a single document, whose brackets balance but which - // cannot be valid JSON: it is not too big, it is invalid. - if (unfinished && find_next_document_index(parser, partial_scalar) != 0) { return TAPE_ERROR; } - if(parser.structural_indexes[0] == 0) { - // If the buffer is partial and we started at index 0 but the document is - // incomplete, it's too big to parse. - return CAPACITY; - } else { - // It is possible that the document could be parsed, we just had a lot - // of white space. - parser.n_structural_indexes = 0; - return EMPTY; - } - } - parser.n_structural_indexes = new_structural_indexes; - } else if (partial == stage1_mode::streaming_final) { - if(have_unclosed_string) { parser.n_structural_indexes--; } - // We truncate the input to the end of the last complete document (or zero). - // Because partial == stage1_mode::streaming_final, it means that we may - // silently ignore trailing garbage. Though it sounds bad, we do it - // deliberately because many people who have streams of JSON documents - // will truncate them for processing. E.g., imagine that you are uncompressing - // the data from a size file or receiving it in chunks from the network. You - // may not know where exactly the last document will be. Meanwhile the - // document_stream instances allow people to know the JSON documents they are - // parsing (see the iterator.source() method). - parser.n_structural_indexes = find_next_document_index(parser); - // We store the initial n_structural_indexes so that the client can see - // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, - // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, - // otherwise, it will copy some prior index. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - // This next line is critical, do not change it unless you understand what you are - // doing. - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - // We tolerate an unclosed string at the very end of the stream. Indeed, users - // often load their data in bulk without being careful and they want us to ignore - // the trailing garbage. - return EMPTY; - } - } else if (partial == stage1_mode::json_sequence_partial) { - // RFC 7464: use RS positions for batch boundaries - // A discarded unclosed string also caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - // The next batch starts at an RS, which the On-Demand API never reads as a - // string, a container or a separator. But an invalid last document may make - // it skip that RS as a value: repeat it, so that the index past it is not - // the next record, which can be a string that does not close in this batch. - parser.structural_indexes[parser.n_structural_indexes + 1] = next_batch_start; - } else if (partial == stage1_mode::json_sequence_final) { - // RFC 7464: final batch, last document extends to EOF - // As above: a discarded unclosed string caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); - // The filter compacted structural_indexes in place and restored the EOF - // sentinel past the compacted end, so the copy below is either the start - // of a truncated document or len, as in streaming_final. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } else if (partial == stage1_mode::comma_delimited_partial) { - // Comma-delimited: filter root-level commas, use comma positions for batch boundaries - if(have_unclosed_string) { - parser.n_structural_indexes--; - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - } else if (partial == stage1_mode::comma_delimited_final) { - // Comma-delimited: final batch, last document extends to EOF - if(have_unclosed_string) { parser.n_structural_indexes--; } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } - checker.check_eof(); - return checker.errors(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. -#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H -/* end file generic/stage1/json_structural_indexer.h for lsx */ -/* including generic/stage1/utf8_validator.h for lsx: #include */ -/* begin file generic/stage1/utf8_validator.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace stage1 { - -/** - * Validates that the string is actual UTF-8. - */ -template -bool generic_validate_utf8(const uint8_t * input, size_t length) { - checker c{}; - buf_block_reader<64> reader(input, length); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - c.check_next_input(in); - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - c.check_next_input(in); - reader.advance(); - c.check_eof(); - return c.errors() == error_code::SUCCESS; -} - -bool generic_validate_utf8(const char * input, size_t length) { - return generic_validate_utf8(reinterpret_cast(input),length); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H -/* end file generic/stage1/utf8_validator.h for lsx */ -/* end file generic/stage1/amalgamated.h for lsx */ -/* including generic/stage2/amalgamated.h for lsx: #include */ -/* begin file generic/stage2/amalgamated.h for lsx */ -// Stuff other things depend on -/* including generic/stage2/base.h for lsx: #include */ -/* begin file generic/stage2/base.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace stage2 { - -class json_iterator; -struct tape_builder; -struct tape_writer; - -} // namespace stage2 -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H -/* end file generic/stage2/base.h for lsx */ -/* including generic/stage2/tape_writer.h for lsx: #include */ -/* begin file generic/stage2/tape_writer.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace lsx { -namespace { -namespace stage2 { - -struct tape_writer { - /** The next place to write to tape */ - uint64_t *next_tape_loc; - - /** Write a signed 64-bit value to tape. */ - simdjson_inline void append_s64(int64_t value) noexcept; - - /** Write an unsigned 64-bit value to tape. */ - simdjson_inline void append_u64(uint64_t value) noexcept; - - /** Write a double value to tape. */ - simdjson_inline void append_double(double value) noexcept; - - /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ - simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; - - /** - * Append a tape entry (an 8-bit type,and 56 bits worth of value). - */ - simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; - - /** - * Skip the current tape entry without writing. - * - * Used to skip the start of the container, since we'll come back later to fill it in when the - * container ends. - */ - simdjson_inline void skip() noexcept; - - /** - * Skip the number of tape entries necessary to write a large u64 or i64. - */ - simdjson_inline void skip_large_integer() noexcept; - - /** - * Skip the number of tape entries necessary to write a double. - */ - simdjson_inline void skip_double() noexcept; - - /** - * Write a value to a known location on tape. - * - * Used to go back and write out the start of a container after the container ends. - */ - simdjson_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; - -private: - /** - * Append both the tape entry, and a supplementary value following it. Used for types that need - * all 64 bits, such as double and uint64_t. - */ - template - simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; -}; // struct tape_writer - -simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { - append2(0, value, internal::tape_type::INT64); -} - -simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { - append(0, internal::tape_type::UINT64); - *next_tape_loc = value; - next_tape_loc++; -} - -/** Write a double value to tape. */ -simdjson_inline void tape_writer::append_double(double value) noexcept { - append2(0, value, internal::tape_type::DOUBLE); -} - -simdjson_inline void tape_writer::skip() noexcept { - next_tape_loc++; -} - -simdjson_inline void tape_writer::skip_large_integer() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::skip_double() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { - *next_tape_loc = val | ((uint64_t(char(t))) << 56); - next_tape_loc++; -} - -template -simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { - append(val, t); - static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); - memcpy(next_tape_loc, &val2, sizeof(val2)); - next_tape_loc++; -} - -simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { - tape_loc = val | ((uint64_t(char(t))) << 56); -} - -simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { - // Write to string buffer: [4-byte LE length][digits][null] - uint32_t str_len = uint32_t(len); - memcpy(string_buf, &str_len, sizeof(uint32_t)); - memcpy(string_buf + sizeof(uint32_t), src, len); - string_buf[sizeof(uint32_t) + len] = 0; - // Tape entry: offset into string buffer - // The caller must set the offset relative to doc.string_buf base - append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches - string_buf += sizeof(uint32_t) + len + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H -/* end file generic/stage2/tape_writer.h for lsx */ -/* including generic/stage2/logger.h for lsx: #include */ -/* begin file generic/stage2/logger.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - - -// This is for an internal-only stage 2 specific logger. -// Set LOG_ENABLED = true to log what stage 2 is doing! -namespace simdjson { -namespace lsx { -namespace { -namespace logger { - - static constexpr const char * DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; - -#if SIMDJSON_VERBOSE_LOGGING - static constexpr const bool LOG_ENABLED = true; -#else - static constexpr const bool LOG_ENABLED = false; -#endif - static constexpr const int LOG_EVENT_LEN = 20; - static constexpr const int LOG_BUFFER_LEN = 30; - static constexpr const int LOG_SMALL_BUFFER_LEN = 10; - static constexpr const int LOG_INDEX_LEN = 5; - - static int log_depth; // Not threadsafe. Log only. - - // Helper to turn unprintable or newline characters into spaces - static simdjson_inline char printable_char(char c) { - if (c >= 0x20) { - return c; - } else { - return ' '; - } - } - - // Print the header and set up log_start - static simdjson_inline void log_start() { - if (LOG_ENABLED) { - log_depth = 0; - printf("\n"); - printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); - printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN+2, DASHES, LOG_BUFFER_LEN+2, DASHES, LOG_SMALL_BUFFER_LEN+2, DASHES, 5+2, DASHES); - } - } - - simdjson_unused static simdjson_inline void log_string(const char *message) { - if (LOG_ENABLED) { - printf("%s\n", message); - } - } - - // Logs a single line from the stage 2 DOM parser - template - static simdjson_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { - if (LOG_ENABLED) { - printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); - auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; - auto next_index = structurals.next_structural; - auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); - auto next = &structurals.buf[*next_index]; - { - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i=0;i */ -/* begin file generic/stage2/json_iterator.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace stage2 { - -class json_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - uint32_t depth{0}; - - /** - * Walk the JSON document. - * - * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as - * the first parameter; some callbacks have other parameters as well: - * - * - visit_document_start() - at the beginning. - * - visit_document_end() - at the end (if things were successful). - * - * - visit_array_start() - at the start `[` of a non-empty array. - * - visit_array_end() - at the end `]` of a non-empty array. - * - visit_empty_array() - when an empty array is encountered. - * - * - visit_object_end() - at the start `]` of a non-empty object. - * - visit_object_start() - at the end `]` of a non-empty object. - * - visit_empty_object() - when an empty object is encountered. - * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is - * guaranteed to point at the first quote of the string (`"key"`). - * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. - * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. - * - * - increment_count(iter) - each time a value is found in an array or object. - */ - template - simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; - - /** - * Create an iterator capable of walking a JSON document. - * - * The document must have already passed through stage 1. - */ - simdjson_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); - - /** - * Look at the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *peek(const uint32_t *batch_end = nullptr) const noexcept; - /** - * Advance to the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *advance(const uint32_t *batch_end = nullptr) noexcept; - /** - * Get the remaining length of the document, from the start of the current token. - */ - simdjson_inline size_t remaining_len() const noexcept; - /** - * Check if we are at the end of the document. - * - * If this is true, there are no more tokens. - */ - simdjson_inline bool at_eof() const noexcept; - /** - * Check if we are at the beginning of the document. - */ - simdjson_inline bool at_beginning() const noexcept; - simdjson_inline uint8_t last_structural() const noexcept; - - /** - * Log that a value has been found. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_value(const char *type) const noexcept; - /** - * Log the start of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_start_value(const char *type) const noexcept; - /** - * Log the end of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_end_value(const char *type) const noexcept; - /** - * Log an error. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_error(const char *error) const noexcept; - - template - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; - template - simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; -}; - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { - logger::log_start(); - - // - // Start the document - // - if (at_eof()) { return EMPTY; } - // Only used when BOUNDED (see advance()); a local, so that it can stay in a - // register, and vanishes from the other walks. - const uint32_t *const batch_end = &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - log_start_value("document"); - SIMDJSON_TRY( visitor.visit_document_start(*this) ); - - // - // Read first value - // - { - auto value = advance(batch_end); - - // Make sure the outer object or array is closed before continuing; otherwise, there are ways we - // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 - if (!STREAMING) { - switch (*value) { - case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; - case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; - } - } - - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; - } - } - goto document_end; - -// -// Object parser states -// -object_begin: - log_start_value("object"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = false; - SIMDJSON_TRY( visitor.visit_object_start(*this) ); - - { - auto key = advance(batch_end); - if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.increment_count(*this) ); - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - -object_field: - if (simdjson_unlikely( (*advance(batch_end) != ':') )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -object_continue: - switch (*advance(batch_end)) { - case ',': - SIMDJSON_TRY( visitor.increment_count(*this) ); - { - auto key = advance(batch_end); - if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - goto object_field; - case '}': log_end_value("object"); SIMDJSON_TRY( visitor.visit_object_end(*this) ); goto scope_end; - default: log_error("No comma between object fields"); return TAPE_ERROR; - } - -scope_end: - depth--; - if (depth == 0) { goto document_end; } - if (dom_parser.is_array[depth]) { goto array_continue; } - goto object_continue; - -// -// Array parser states -// -array_begin: - log_start_value("array"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = true; - SIMDJSON_TRY( visitor.visit_array_start(*this) ); - SIMDJSON_TRY( visitor.increment_count(*this) ); - -array_value: - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -array_continue: - switch (*advance(batch_end)) { - case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; - case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; - default: log_error("Missing comma between array values"); return TAPE_ERROR; - } - -document_end: - log_end_value("document"); - SIMDJSON_TRY( visitor.visit_document_end(*this) ); - - dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); - - // If we didn't make it to the end, it's an error - if ( !STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes ) { - log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); - return TAPE_ERROR; - } - - return SUCCESS; - -} // walk_document() - -simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { -} - -// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires -// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). -// -// When BOUNDED (see dom_parser_implementation::_bounded_batch), a document may -// not run past batch_end, one past the last structural index of the batch: -// past it, we return the same dummy, and walk_document fails cleanly. -template -simdjson_inline const uint8_t *json_iterator::peek(const uint32_t *batch_end) const noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -template -simdjson_inline const uint8_t *json_iterator::advance(const uint32_t *batch_end) noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural++); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -simdjson_inline size_t json_iterator::remaining_len() const noexcept { - return dom_parser.len - *(next_structural-1); -} - -simdjson_inline bool json_iterator::at_eof() const noexcept { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; -} -simdjson_inline bool json_iterator::at_beginning() const noexcept { - return next_structural == dom_parser.structural_indexes.get(); -} -simdjson_inline uint8_t json_iterator::last_structural() const noexcept { - return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; -} - -simdjson_inline void json_iterator::log_value(const char *type) const noexcept { - logger::log_line(*this, "", type, ""); -} - -simdjson_inline void json_iterator::log_start_value(const char *type) const noexcept { - logger::log_line(*this, "+", type, ""); - if (logger::LOG_ENABLED) { logger::log_depth++; } -} - -simdjson_inline void json_iterator::log_end_value(const char *type) const noexcept { - if (logger::LOG_ENABLED) { logger::log_depth--; } - logger::log_line(*this, "-", type, ""); -} - -simdjson_inline void json_iterator::log_error(const char *error) const noexcept { - logger::log_line(*this, "", "ERROR", error); -} - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { - switch (*value) { - case '"': return visitor.visit_root_string(*this, value); - case 't': return visitor.visit_root_true_atom(*this, value); - case 'f': return visitor.visit_root_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_root_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_root_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_root_inf_atom(*this, value); -#else - case 'n': return visitor.visit_root_null_atom(*this, value); -#endif - case '-': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return visitor.visit_root_number(*this, value); - default: - log_error("Document starts with a non-value character"); - return TAPE_ERROR; - } -} -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { - // Use the fact that most scalars are going to be either strings or numbers. - if(*value == '"') { - return visitor.visit_string(*this, value); - } else if (((*value - '0') < 10) || (*value == '-')) { - return visitor.visit_number(*this, value); - } - // true, false, null are uncommon. - switch (*value) { - case 't': return visitor.visit_true_atom(*this, value); - case 'f': return visitor.visit_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_inf_atom(*this, value); -#else - case 'n': return visitor.visit_null_atom(*this, value); -#endif - default: - log_error("Non-value found when value was expected!"); - return TAPE_ERROR; - } -} - -} // namespace stage2 -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H -/* end file generic/stage2/json_iterator.h for lsx */ -/* including generic/stage2/stringparsing.h for lsx: #include */ -/* begin file generic/stage2/stringparsing.h for lsx */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace lsx { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - // Decode adjacent Unicode escapes without returning to the - // quote-and-backslash scanner between code points. - do { - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } while (src[0] == '\\' && src[1] == 'u'); - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -/** - * Bounds-safe variant of parse_string for input buffers that are NOT padded to - * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input - * byte (buf + len). It behaves exactly like parse_string while we are at least - * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays - * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few - * remaining bytes into a space-padded scratch buffer and finishes with the - * regular parse_string. This keeps the delicate escape/Unicode handling in one - * place (the proven parse_string) rather than duplicating it. - * - * Correctness relies on stage 1 having validated the string, i.e. there is an - * unescaped closing quote within [src, buf_end); that quote is therefore inside - * the copied scratch, so parse_string finds it without running off the scratch. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { - // Far from the end: identical to parse_string's loop. The guard uses - // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past - // buf_end; escape/Unicode look-aheads read within the string (before the - // closing quote, which is < buf_end), so they are in bounds here too. - // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: - // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate - // following bytes (+6 from the '\'), then (if it sees \u) another - // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the - // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the - // chunk start. The +12 margin ensures that even on kernels where - // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays - // in-bounds. The scratch fallback (3*PAD) is already safe. - while (src + SIMDJSON_PADDING + 12 <= buf_end) { - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - if (bs_quote.has_quote_first()) { - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - if (escape_char == 'u') { - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - // Within the final SIMDJSON_PADDING bytes: copy what remains into a - // space-padded scratch (spaces are neither quote nor backslash, so they do not - // disturb matching) and let the regular parser finish from there. The closing - // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in - // the chunk starting at some offset <= remaining and reads at most - // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. - // We size at 3x for a comfortable margin (the unicode look-ahead reads a few - // extra bytes past an escape). - uint8_t scratch[SIMDJSON_PADDING * 3]; - const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING - std::memset(scratch, ' ', sizeof(scratch)); - std::memcpy(scratch, src, remaining); - return parse_string(scratch, dst, allow_replacement); -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for lsx */ -/* including generic/stage2/tape_builder.h for lsx: #include */ -/* begin file generic/stage2/tape_builder.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - - -namespace simdjson { -namespace lsx { -namespace { -namespace stage2 { - -template -struct tape_builder_impl { - /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; - /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator &iter) noexcept; - - /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator &iter) noexcept; - /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator &iter) noexcept; - /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator &iter) noexcept; - - /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator &iter) noexcept; - /** - * Called when a key in a field is encountered. - * - * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array - * will be called after this with the field value. - */ - simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; - /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator &iter) noexcept; - /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator &iter) noexcept; - - /** - * Called when a string, number, boolean or null is found. - */ - simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; - /** - * Called when a string, number, boolean or null is found at the top level of a document (i.e. - * when there is no array or object and the entire document is a single string, number, boolean or - * null. - * - * This is separate from primitive() because simdjson's normal primitive parsing routines assume - * there is at least one more token after the value, which is only true in an array or object. - */ - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - -#if SIMDJSON_ENABLE_NAN_INF - simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, - // this returns a tape error on failure. - simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; -#endif - - /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; - - /** Next location to write to tape */ - tape_writer tape; -public: - simdjson_inline tape_builder_impl(dom::document &doc) noexcept; -private: - /** Next write location in the string buf for stage 2 parsing */ - uint8_t *current_string_buf_loc; - - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - simdjson_inline void start_container(json_iterator &iter) noexcept; - simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder_impl - -// Thin, non-templated entry so each architecture's stage2() keeps calling -// tape_builder::parse_document unchanged. It chooses the bounds-safe -// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the -// choice is a compile-time constant inside the walk: the padded path carries no -// extra branch or load (see tape_builder_impl::visit_string). -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } - - // The stage 2 entry points of dom::document_stream, which picks one per batch - // (see internal::dom_parser_implementation::stage2_next_function). Out of - // line, so that stage2_next() can share the code. - simdjson_warn_unused static simdjson_never_inline error_code parse_next_document( - internal::dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - return parse_document(static_cast(dom_parser), doc); - } - - // A document of a stream batch that stage 2 must bound to its structural - // indexes (see internal::dom_parser_implementation::_bounded_batch). - simdjson_warn_unused static simdjson_never_inline error_code parse_next_bounded_document( - internal::dom_parser_implementation &_dom_parser, dom::document &doc) noexcept { - dom_parser_implementation &dom_parser = static_cast(_dom_parser); - dom_parser.doc = &doc; - json_iterator iter(dom_parser, dom_parser.next_structural_index); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } -}; - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_root_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { - constexpr uint32_t start_tape_index = 0; - tape.append(start_tape_index, internal::tape_type::ROOT); - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { - return visit_string(iter, key, true); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 - return SUCCESS; -} - -template -simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { - iter.log_value(key ? "key" : "string"); - uint8_t *dst = on_start_string(iter); - // We do not allow replacement when the escape characters are invalid. - // UNPADDED is a compile-time constant chosen once per document by - // tape_builder::parse_document, so the padded build instantiates only the - // plain parse_string call below -- no runtime branch and no flag load. - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); - } else { - dst = stringparsing::parse_string(value+1, dst, false); - } - if (dst == nullptr) { - iter.log_error("Invalid escape in string"); - return STRING_ERROR; - } - on_end_string(dst); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { - return visit_string(iter, value); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("number"); - const uint8_t *num = value; - std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - // numberparsing reads ahead in 8-byte blocks for floats - // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a - // number whose digits reach the final bytes of an unpadded buffer would read - // past it. *(next_structural) is the offset of the token following this - // number, hence an upper bound on where the digits end; when that is within - // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail - // (mirroring visit_root_number). This fires only for numbers near the end. - if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { - const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document - copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, rl); - std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); - num = copy.get(); - } - } - error_code err = numberparsing::parse_number(num, tape); - if (simdjson_unlikely(err == BIGINT_ERROR && - iter.dom_parser._number_as_string)) { - // Write big integer to string buffer using the same format as strings. - // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = num; - if (*p == '-') p++; - while (numberparsing::is_digit(*p)) p++; - // The digit run must be terminated by a structural or whitespace character; otherwise the - // token is malformed (e.g. "123456789123456789123x"). - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - num); - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); - uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, num, len); - dst += len; - on_end_string(dst); - return SUCCESS; - } - return err; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { - // - // We need to make a copy to make sure that the string is space terminated. - // This is not about padding the input, which should already padded up - // to len + SIMDJSON_PADDING. However, we have no control at this stage - // on how the padding was done. What if the input string was padded with nulls? - // It is quite common for an input string to have an extra null character (C string). - // We do not want to allow 9\0 (where \0 is the null character) inside a JSON - // document, but the string "9\0" by itself is fine. So we make a copy and - // pad the input with spaces when we know that there is just one input element. - // This copy is relatively expensive, but it will almost never be called in - // practice unless you are in the strange scenario where you have many JSON - // documents made of single atoms. - // - // In a stream, the input goes on with other documents: copy up to the next - // structural only, not to the end of the batch. - const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); - std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, len); - std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); - error_code error = visit_number(iter, copy.get()); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated - // token at the very end of an unpadded buffer would over-read. Use the - // length-aware form there (the root variant already does this). - const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) - : atomparsing::is_valid_true_atom(value); - if (!ok) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) - : atomparsing::is_valid_false_atom(value); - if (!ok) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) - : atomparsing::is_valid_null_atom(value); - if (!ok) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -#if SIMDJSON_ENABLE_NAN_INF -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - // For unpadded input use the length-aware validator so the 'infinity'-style - // 8-byte compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) - : atomparsing::is_valid_nan_atom(value); - if (!ok) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. - // For unpadded input use the length-aware validator so the 'infinity' 8-byte - // compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) - : atomparsing::is_valid_inf_atom(value); - if (!ok) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure - if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -// private: - -template -simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { - return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - auto start_index = next_tape_index(iter); - tape.append(start_index+2, start); - tape.append(start_index, end); - return SUCCESS; -} - -template -simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); - iter.dom_parser.open_containers[iter.depth].count = 0; - tape.skip(); // We don't actually *write* the start element until the end. -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - // Write the ending tape element, pointing at the start location - const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; - tape.append(start_tape_index, end); - // Write the start tape element, pointing at the end location (and including count) - // count can overflow if it exceeds 24 bits... so we saturate - // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). - const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; - const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); - return SUCCESS; -} - -template -simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { - // we advance the point, accounting for the fact that we have a NULL termination - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); - return current_string_buf_loc + sizeof(uint32_t); -} - -template -simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { - uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); - // TODO check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); - // NULL termination is still handy if you expect all your strings to - // be NULL terminated? It comes at a small cost - *dst = 0; - current_string_buf_loc = dst + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H -/* end file generic/stage2/tape_builder.h for lsx */ -/* end file generic/stage2/amalgamated.h for lsx */ - -// -// Stage 1 -// -namespace simdjson { -namespace lsx { - -simdjson_warn_unused error_code implementation::create_dom_parser_implementation( - size_t capacity, - size_t max_depth, - std::unique_ptr& dst -) const noexcept { - dst.reset( new (std::nothrow) dom_parser_implementation() ); - if (!dst) { return MEMALLOC; } - if (auto err = dst->set_capacity(capacity)) - return err; - if (auto err = dst->set_max_depth(max_depth)) - return err; - return SUCCESS; -} - -namespace { - -using namespace simd; - -simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - // Inspired by haswell. - // LSX use low 5 bits as index. For the 6 operators (:,[]{}), the unique-5bits is [6:2]. - // The ASCII white-space and operators have these values: (char, hex, unique-5bits) - // (' ', 20, 00000) ('\t', 09, 01001) ('\n', 0A, 01010) ('\r', 0D, 01101) - // (',', 2C, 01011) (':', 3A, 01110) ('[', 5B, 10110) ('{', 7B, 11110) (']', 5D, 10111) ('}', 7D, 11111) - const simd8 ws_table = simd8::repeat_16( - ' ', 0, 0, 0, 0, 0, 0, 0, 0, '\t', '\n', 0, 0, '\r', 0, 0 - ); - const simd8 op_table_lo = simd8::repeat_16( - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ',', 0, 0, ':', 0 - ); - const simd8 op_table_hi = simd8::repeat_16( - 0, 0, 0, 0, 0, 0, '[', ']', 0, 0, 0, 0, 0, 0, '{', '}' - ); - uint64_t ws = in.eq({ - in.chunks[0].lookup_16(ws_table), - in.chunks[1].lookup_16(ws_table), - in.chunks[2].lookup_16(ws_table), - in.chunks[3].lookup_16(ws_table) - }); - uint64_t op = in.eq({ - __lsx_vshuf_b(op_table_hi, op_table_lo, in.chunks[0].shr<2>()), - __lsx_vshuf_b(op_table_hi, op_table_lo, in.chunks[1].shr<2>()), - __lsx_vshuf_b(op_table_hi, op_table_lo, in.chunks[2].shr<2>()), - __lsx_vshuf_b(op_table_hi, op_table_lo, in.chunks[3].shr<2>()) - }); - - return { ws, op }; -} - -simdjson_inline bool is_ascii(const simd8x64& input) { - return input.reduce_or().is_ascii(); -} - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be >= 0x80 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be >= 0x80 - return is_third_byte | is_fourth_byte; -} - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -// -// Stage 2 -// - -// -// Implementation-specific overrides -// -namespace simdjson { -namespace lsx { - -simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { - return lsx::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, stage1_mode streaming) noexcept { - this->buf = _buf; - this->len = _len; - return lsx::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); -} - -simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return lsx::stage1::generic_validate_utf8(buf,len); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_next_document(*this, _doc); -} - -internal::dom_parser_implementation::stage2_next_function_t dom_parser_implementation::stage2_next_function(bool bounded) const noexcept { - return bounded ? stage2::tape_builder::parse_next_bounded_document : stage2::tape_builder::parse_next_document; -} - -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { - return lsx::stringparsing::parse_string(src, dst, allow_replacement); -} - -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { - return lsx::stringparsing::parse_wobbly_string(src, dst); -} - -simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { - auto error = stage1(_buf, _len, stage1_mode::regular); - if (error) { return error; } - return stage2(_doc); -} - -} // namespace lsx -} // namespace simdjson - -/* including simdjson/lsx/end.h: #include */ -/* begin file simdjson/lsx/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT -/* undefining SIMDJSON_IMPLEMENTATION from "lsx" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/lsx/end.h */ - -#endif // SIMDJSON_SRC_LSX_CPP -/* end file lsx.cpp */ -#endif -#if SIMDJSON_IMPLEMENTATION_RVV_VLS -/* including rvv-vls.cpp: #include */ -/* begin file rvv-vls.cpp */ -#ifndef SIMDJSON_SRC_RVV_VLS_CPP -#define SIMDJSON_SRC_RVV_VLS_CPP - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* including simdjson/rvv-vls.h: #include */ -/* begin file simdjson/rvv-vls.h */ -#ifndef SIMDJSON_RVV_VLS_H -#define SIMDJSON_RVV_VLS_H - - -/* including simdjson/rvv-vls/begin.h: #include "simdjson/rvv-vls/begin.h" */ -/* begin file simdjson/rvv-vls/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "rvv_vls" */ -#define SIMDJSON_IMPLEMENTATION rvv_vls -/* including simdjson/rvv-vls/base.h: #include "simdjson/rvv-vls/base.h" */ -/* begin file simdjson/rvv-vls/base.h */ -#ifndef SIMDJSON_RVV_VLS_BASE_H -#define SIMDJSON_RVV_VLS_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * RVV-VLS implementation. - */ -namespace rvv_vls { - -class implementation; - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_BASE_H -/* end file simdjson/rvv-vls/base.h */ -/* including simdjson/rvv-vls/intrinsics.h: #include "simdjson/rvv-vls/intrinsics.h" */ -/* begin file simdjson/rvv-vls/intrinsics.h */ -#ifndef SIMDJSON_RVV_VLS_INTRINSICS_H -#define SIMDJSON_RVV_VLS_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#define simdutf_vrgather_u8m1x2(tbl, idx) \ - __riscv_vcreate_v_u8m1_u8m2( \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m2_u8m1(idx, 0), \ - __riscv_vsetvlmax_e8m1()), \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m2_u8m1(idx, 1), \ - __riscv_vsetvlmax_e8m1())) - -#define simdutf_vrgather_u8m1x4(tbl, idx) \ - __riscv_vcreate_v_u8m1_u8m4( \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m4_u8m1(idx, 0), \ - __riscv_vsetvlmax_e8m1()), \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m4_u8m1(idx, 1), \ - __riscv_vsetvlmax_e8m1()), \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m4_u8m1(idx, 2), \ - __riscv_vsetvlmax_e8m1()), \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m4_u8m1(idx, 3), \ - __riscv_vsetvlmax_e8m1())) - -#if __riscv_zbc -#include -#endif - -#endif // SIMDJSON_RVV_VLS_INTRINSICS_H -/* end file simdjson/rvv-vls/intrinsics.h */ -/* including simdjson/rvv-vls/bitmanipulation.h: #include "simdjson/rvv-vls/bitmanipulation.h" */ -/* begin file simdjson/rvv-vls/bitmanipulation.h */ -#ifndef SIMDJSON_RVV_VLS_BITMANIPULATION_H -#define SIMDJSON_RVV_VLS_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { - return __builtin_ctzll(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { - return __builtin_clzll(input_num); -} - -simdjson_inline long long int count_ones(uint64_t input_num) { - return __builtin_popcountll(input_num); -} - - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_BITMANIPULATION_H -/* end file simdjson/rvv-vls/bitmanipulation.h */ -/* including simdjson/rvv-vls/bitmask.h: #include "simdjson/rvv-vls/bitmask.h" */ -/* begin file simdjson/rvv-vls/bitmask.h */ -#ifndef SIMDJSON_RVV_VLS_BITMASK_H -#define SIMDJSON_RVV_VLS_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { -#if __riscv_zbc - return __riscv_clmul_64(bitmask, ~(uint64_t)0); -#elif __riscv_zvbc - return __riscv_vmv_x(__riscv_vclmul(__riscv_vmv_s_x_u64m1(bitmask, 1), ~(uint64_t)0, 1)); -#else - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; -#endif - return bitmask; -} - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_BITMASK_H - -/* end file simdjson/rvv-vls/bitmask.h */ -/* including simdjson/rvv-vls/simd.h: #include "simdjson/rvv-vls/simd.h" */ -/* begin file simdjson/rvv-vls/simd.h */ -#ifndef SIMDJSON_RVV_VLS_SIMD_H -#define SIMDJSON_RVV_VLS_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace simd { - -#if __riscv_v_fixed_vlen >= 512 - static constexpr size_t VL8 = 512/8; - using vint8_t = vint8m1_t __attribute__((riscv_rvv_vector_bits(512))); - using vuint8_t = vuint8m1_t __attribute__((riscv_rvv_vector_bits(512))); - using vbool_t = vbool8_t __attribute__((riscv_rvv_vector_bits(512/8))); - using vbitmask_t = uint64_t; -#else - static constexpr size_t VL8 = __riscv_v_fixed_vlen/8; - using vint8_t = vint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen))); - using vuint8_t = vuint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen))); - using vbool_t = vbool8_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen/8))); - #if __riscv_v_fixed_vlen == 128 - using vbitmask_t = uint16_t; - #elif __riscv_v_fixed_vlen == 256 - using vbitmask_t = uint32_t; - #endif -#endif - -#if __riscv_v_fixed_vlen == 128 - using vuint8x64_t = vuint8m4_t __attribute__((riscv_rvv_vector_bits(512))); - using vboolx64_t = vbool2_t __attribute__((riscv_rvv_vector_bits(512/8))); -#elif __riscv_v_fixed_vlen == 256 - using vuint8x64_t = vuint8m2_t __attribute__((riscv_rvv_vector_bits(512))); - using vboolx64_t = vbool4_t __attribute__((riscv_rvv_vector_bits(512/8))); -#else - using vuint8x64_t = vuint8m1_t __attribute__((riscv_rvv_vector_bits(512))); - using vboolx64_t = vbool8_t __attribute__((riscv_rvv_vector_bits(512/8))); -#endif - - template - struct simd8; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8 { - vbool_t value; - using bitmask_t = vbitmask_t; - static constexpr int SIZE = sizeof(value); - - simdjson_inline simd8(const vbool_t _value) : value(_value) {} - simdjson_inline simd8() : simd8(__riscv_vmclr_m_b8(VL8)) {} - simdjson_inline simd8(bool _value) : simd8(splat(_value)) {} - - simdjson_inline operator const vbool_t&() const { return value; } - simdjson_inline operator vbool_t&() { return value; } - - static simdjson_inline simd8 splat(bool _value) { - return __riscv_vreinterpret_b8(__riscv_vmv_v_x_u64m1(((uint64_t)!_value)-1, 1)); - } - - simdjson_inline vbitmask_t to_bitmask() const { -#if __riscv_v_fixed_vlen == 128 - return __riscv_vmv_x(__riscv_vreinterpret_u16m1(value)); -#elif __riscv_v_fixed_vlen == 256 - return __riscv_vmv_x(__riscv_vreinterpret_u32m1(value)); -#else - return __riscv_vmv_x(__riscv_vreinterpret_u64m1(value)); -#endif - } - - // Bit operations - simdjson_inline simd8 operator|(const simd8 other) const { return __riscv_vmor(*this, other, VL8); } - simdjson_inline simd8 operator&(const simd8 other) const { return __riscv_vmand(*this, other, VL8); } - simdjson_inline simd8 operator^(const simd8 other) const { return __riscv_vmxor(*this, other, VL8); } - simdjson_inline simd8 bit_andnot(const simd8 other) const { return __riscv_vmandn(other, *this, VL8); } - simdjson_inline simd8 operator~() const { return __riscv_vmnot(*this, VL8); } - simdjson_inline simd8& operator|=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline simd8& operator&=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline simd8& operator^=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Unsigned bytes - template<> - struct simd8 { - - vuint8_t value; - static constexpr int SIZE = sizeof(value); - - simdjson_inline simd8(const vuint8_t _value) : value(_value) {} - simdjson_inline simd8() : simd8(zero()) {} - simdjson_inline simd8(const uint8_t values[VL8]) : simd8(load(values)) {} - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - simdjson_inline simd8(simd8 mask) : value(__riscv_vmerge_vxm_u8m1(zero(), -1, (vbool_t)mask, VL8)) {} - - simdjson_inline operator const vuint8_t&() const { return this->value; } - simdjson_inline operator vuint8_t&() { return this->value; } - - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(vuint8_t{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - static simdjson_inline vuint8_t splat(uint8_t _value) { return __riscv_vmv_v_x_u8m1(_value, VL8); } - static simdjson_inline vuint8_t zero() { return splat(0); } - static simdjson_inline vuint8_t load(const uint8_t values[VL8]) { return __riscv_vle8_v_u8m1(values, VL8); } - - // Bit operations - simdjson_inline simd8 operator|(const simd8 other) const { return __riscv_vor_vv_u8m1( value, other, VL8); } - simdjson_inline simd8 operator&(const simd8 other) const { return __riscv_vand_vv_u8m1( value, other, VL8); } - simdjson_inline simd8 operator^(const simd8 other) const { return __riscv_vxor_vv_u8m1( value, other, VL8); } - simdjson_inline simd8 operator~() const { return __riscv_vnot_v_u8m1(value, VL8); } -#if __riscv_zvbb - simdjson_inline simd8 bit_andnot(const simd8 other) const { return __riscv_vandn_vv_u8m1(other, value, VL8); } -#else - simdjson_inline simd8 bit_andnot(const simd8 other) const { return other & ~*this; } -#endif - simdjson_inline simd8& operator|=(const simd8 other) { value = *this | other; return *this; } - simdjson_inline simd8& operator&=(const simd8 other) { value = *this & other; return *this; } - simdjson_inline simd8& operator^=(const simd8 other) { value = *this ^ other; return *this; } - - simdjson_inline simd8 operator==(const simd8 other) const { return __riscv_vmseq(value, other, VL8); } - simdjson_inline simd8 operator==(uint8_t other) const { return __riscv_vmseq(value, other, VL8); } - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return __riscv_vslideup(__riscv_vslidedown(prev_chunk, VL8-N, VL8), value, N, VL8); - } - - // Store to array - simdjson_inline void store(uint8_t dst[VL8]) const { return __riscv_vse8(dst, value, VL8); } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return __riscv_vsaddu(value, other, VL8); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return __riscv_vssubu(value, other, VL8); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return __riscv_vadd(value, other, VL8); } - simdjson_inline simd8 operator-(const simd8 other) const { return __riscv_vsub(value, other, VL8); } - simdjson_inline simd8& operator+=(const simd8 other) { value = *this + other; return *this; } - simdjson_inline simd8& operator-=(const simd8 other) { value = *this - other; return *this; } - - // Order-specific operations - simdjson_inline simd8 operator<=(const simd8 other) const { return __riscv_vmsleu(value, other, VL8); } - simdjson_inline simd8 operator>=(const simd8 other) const { return __riscv_vmsgeu(value, other, VL8); } - simdjson_inline simd8 operator<(const simd8 other) const { return __riscv_vmsltu(value, other, VL8); } - simdjson_inline simd8 operator>(const simd8 other) const { return __riscv_vmsgtu(value, other, VL8); } - - // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. - simdjson_inline simd8 gt_bits(const simd8 other) const { return simd8(*this > other); } - // Same as <, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. - simdjson_inline simd8 lt_bits(const simd8 other) const { return simd8(*this < other); } - - // Bit-specific operations - simdjson_inline bool any_bits_set_anywhere() const { - return __riscv_vfirst(__riscv_vmsne(value, 0, VL8), VL8) >= 0; - } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } - template - simdjson_inline simd8 shr() const { return __riscv_vsrl(value, N, VL8); } - template - simdjson_inline simd8 shl() const { return __riscv_vsll(value, N, VL8); } - - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return __riscv_vrgather(lookup_table, value, VL8); - } - - // compress inactive elements, to match AVX-512 behavior - template - simdjson_inline void compress(vbitmask_t mask, L * output) const { - mask = (vbitmask_t)~mask; -#if __riscv_v_fixed_vlen == 128 - vbool8_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u16m1(mask, 1)); -#elif __riscv_v_fixed_vlen == 256 - vbool8_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u32m1(mask, 1)); -#else - vbool8_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u64m1(mask, 1)); -#endif - __riscv_vse8_v_u8m1(output, __riscv_vcompress(value, m, VL8), count_ones(mask)); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 { - vint8_t value; - static constexpr int SIZE = sizeof(value); - - simdjson_inline simd8(const vint8_t _value) : value(_value) {} - simdjson_inline simd8() : simd8(zero()) {} - simdjson_inline simd8(const int8_t values[VL8]) : simd8(load(values)) {} - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - - simdjson_inline operator const vint8_t&() const { return this->value; } - simdjson_inline operator vint8_t&() { return this->value; } - - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(vint8_t{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - static simdjson_inline vint8_t splat(int8_t _value) { return __riscv_vmv_v_x_i8m1(_value, VL8); } - static simdjson_inline vint8_t zero() { return splat(0); } - static simdjson_inline vint8_t load(const int8_t values[VL8]) { return __riscv_vle8_v_i8m1(values, VL8); } - - - simdjson_inline void store(int8_t dst[VL8]) const { return __riscv_vse8(dst, value, VL8); } - - // Explicit conversion to/from unsigned - simdjson_inline explicit simd8(const vuint8_t other): simd8(__riscv_vreinterpret_i8m1(other)) {} - simdjson_inline explicit operator simd8() const { return __riscv_vreinterpret_u8m1(value); } - - // Math - simdjson_inline simd8 operator+(const simd8 other) const { return __riscv_vadd(value, other, VL8); } - simdjson_inline simd8 operator-(const simd8 other) const { return __riscv_vsub(value, other, VL8); } - simdjson_inline simd8& operator+=(const simd8 other) { value = *this + other; return *this; } - simdjson_inline simd8& operator-=(const simd8 other) { value = *this - other; return *this; } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val( const simd8 other) const { return __riscv_vmax( value, other, VL8); } - simdjson_inline simd8 min_val( const simd8 other) const { return __riscv_vmin( value, other, VL8); } - simdjson_inline simd8 operator>( const simd8 other) const { return __riscv_vmsgt(value, other, VL8); } - simdjson_inline simd8 operator<( const simd8 other) const { return __riscv_vmslt(value, other, VL8); } - simdjson_inline simd8 operator==(const simd8 other) const { return __riscv_vmseq(value, other, VL8); } - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return __riscv_vslideup(__riscv_vslidedown(prev_chunk, VL8-N, VL8), value, N, VL8); - } - - // Perform a lookup assuming no value is larger than 16 - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return __riscv_vrgather(lookup_table, value, VL8); - } - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - template - struct simd8x64; - template<> - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - vuint8x64_t value; - -#if __riscv_v_fixed_vlen >= 512 - template simd8 get() const { return value; } -#else - template simd8 get() const { return __riscv_vget_u8m1(value, idx); } -#endif - - simdjson_inline operator const vuint8x64_t&() const { return this->value; } - simdjson_inline operator vuint8x64_t&() { return this->value; } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - -#if __riscv_v_fixed_vlen == 128 - simdjson_inline simd8x64(const uint8_t *ptr, size_t n = 64) : value(__riscv_vle8_v_u8m4(ptr, n)) {} -#elif __riscv_v_fixed_vlen == 256 - simdjson_inline simd8x64(const uint8_t *ptr, size_t n = 64) : value(__riscv_vle8_v_u8m2(ptr, n)) {} -#else - simdjson_inline simd8x64(const uint8_t *ptr, size_t n = 64) : value(__riscv_vle8_v_u8m1(ptr, n)) {} -#endif - - simdjson_inline void store(uint8_t ptr[64]) const { - __riscv_vse8(ptr, value, 64); - } - - simdjson_inline bool is_ascii() const { -#if __riscv_v_fixed_vlen == 128 - return __riscv_vfirst(__riscv_vmslt(__riscv_vreinterpret_i8m4(value), 0, 64), 64) < 0; -#elif __riscv_v_fixed_vlen == 256 - return __riscv_vfirst(__riscv_vmslt(__riscv_vreinterpret_i8m2(value), 0, 64), 64) < 0; -#else - return __riscv_vfirst(__riscv_vmslt(__riscv_vreinterpret_i8m1(value), 0, 64), 64) < 0; -#endif - } - - // compress inactive elements, to match AVX-512 behavior - simdjson_inline uint64_t compress(uint64_t mask, uint8_t * output) const { - mask = ~mask; -#if __riscv_v_fixed_vlen == 128 - vboolx64_t m = __riscv_vreinterpret_b2(__riscv_vmv_s_x_u64m1(mask, 1)); -#elif __riscv_v_fixed_vlen == 256 - vboolx64_t m = __riscv_vreinterpret_b4(__riscv_vmv_s_x_u64m1(mask, 1)); -#else - vboolx64_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u64m1(mask, 1)); -#endif - size_t cnt = count_ones(mask); - __riscv_vse8(output, __riscv_vcompress(value, m, 64), cnt); - return cnt; - } - - simdjson_inline uint64_t eq(const uint8_t m) const { - return __riscv_vmv_x(__riscv_vreinterpret_u64m1(__riscv_vmseq(value, m, 64))); - } - - simdjson_inline uint64_t lteq(const uint8_t m) const { - return __riscv_vmv_x(__riscv_vreinterpret_u64m1(__riscv_vmsleu(value, m, 64))); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_SIMD_H -/* end file simdjson/rvv-vls/simd.h */ -/* including simdjson/rvv-vls/stringparsing_defs.h: #include "simdjson/rvv-vls/stringparsing_defs.h" */ -/* begin file simdjson/rvv-vls/stringparsing_defs.h */ -#ifndef SIMDJSON_RVV_VLS_STRINGPARSING_DEFS_H -#define SIMDJSON_RVV_VLS_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint64_t BYTES_PROCESSED = sizeof(simd8); - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint64_t bs_bits; - uint64_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - v.store(dst); - return { (v == '\\').to_bitmask(), (v == '"').to_bitmask() }; -} - - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_STRINGPARSING_DEFS_H -/* end file simdjson/rvv-vls/stringparsing_defs.h */ -/* including simdjson/rvv-vls/numberparsing_defs.h: #include "simdjson/rvv-vls/numberparsing_defs.h" */ -/* begin file simdjson/rvv-vls/numberparsing_defs.h */ -#ifndef SIMDJSON_RVV_VLS_NUMBERPARSING_DEFS_H -#define SIMDJSON_RVV_VLS_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#ifdef JSON_TEST_NUMBERS // for unit testing -void found_invalid_number(const uint8_t *buf); -void found_integer(int64_t result, const uint8_t *buf); -void found_unsigned_integer(uint64_t result, const uint8_t *buf); -void found_float(double result, const uint8_t *buf); -#endif - -namespace simdjson { -namespace rvv_vls { -namespace numberparsing { - -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const char *chars) { - uint64_t val; -#if __riscv_misaligned_fast - memcpy(&val, chars, sizeof(uint64_t)); -#else - val = __riscv_vmv_x(__riscv_vreinterpret_u64m1(__riscv_vlmul_ext_u8m1(__riscv_vle8_v_u8mf2((uint8_t*)chars, 8)))); -#endif - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - return parse_eight_digits_unrolled(reinterpret_cast(chars)); -} - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); - return answer; -} - -} // namespace numberparsing -} // namespace rvv_vls -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_RVV_VLS_NUMBERPARSING_DEFS_H -/* end file simdjson/rvv-vls/numberparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 -/* end file simdjson/rvv-vls/begin.h */ -/* including simdjson/generic/amalgamated.h for rvv_vls: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for rvv_vls */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for rvv_vls: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for rvv_vls */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for rvv_vls */ -/* including simdjson/generic/jsoncharutils.h for rvv_vls: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for rvv_vls */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for rvv_vls */ -/* including simdjson/generic/atomparsing.h for rvv_vls: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for rvv_vls */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace rvv_vls { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for rvv_vls */ -/* including simdjson/generic/dom_parser_implementation.h for rvv_vls: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for rvv_vls */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace rvv_vls -} // namespace simdjson - -namespace simdjson { -namespace rvv_vls { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for rvv_vls */ -/* including simdjson/generic/implementation_simdjson_result_base.h for rvv_vls: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for rvv_vls */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for rvv_vls */ -/* including simdjson/generic/numberparsing.h for rvv_vls: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for rvv_vls */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace rvv_vls { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for rvv_vls */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for rvv_vls: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for rvv_vls */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for rvv_vls */ -/* end file simdjson/generic/amalgamated.h for rvv_vls */ -/* including simdjson/rvv-vls/end.h: #include "simdjson/rvv-vls/end.h" */ -/* begin file simdjson/rvv-vls/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* undefining SIMDJSON_IMPLEMENTATION from "rvv_vls" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/rvv-vls/end.h */ - -#endif // SIMDJSON_RVV_VLS_H -/* end file simdjson/rvv-vls.h */ -/* including simdjson/rvv-vls/implementation.h: #include */ -/* begin file simdjson/rvv-vls/implementation.h */ -#ifndef SIMDJSON_RVV_VLS_IMPLEMENTATION_H -#define SIMDJSON_RVV_VLS_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation( - "rvv_vls", - "RISC-V V extension", - internal::instruction_set::RVV_VLS - ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_IMPLEMENTATION_H -/* end file simdjson/rvv-vls/implementation.h */ - -/* including simdjson/rvv-vls/begin.h: #include */ -/* begin file simdjson/rvv-vls/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "rvv_vls" */ -#define SIMDJSON_IMPLEMENTATION rvv_vls -/* including simdjson/rvv-vls/base.h: #include "simdjson/rvv-vls/base.h" */ -/* begin file simdjson/rvv-vls/base.h */ -#ifndef SIMDJSON_RVV_VLS_BASE_H -#define SIMDJSON_RVV_VLS_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * RVV-VLS implementation. - */ -namespace rvv_vls { - -class implementation; - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_BASE_H -/* end file simdjson/rvv-vls/base.h */ -/* including simdjson/rvv-vls/intrinsics.h: #include "simdjson/rvv-vls/intrinsics.h" */ -/* begin file simdjson/rvv-vls/intrinsics.h */ -#ifndef SIMDJSON_RVV_VLS_INTRINSICS_H -#define SIMDJSON_RVV_VLS_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#define simdutf_vrgather_u8m1x2(tbl, idx) \ - __riscv_vcreate_v_u8m1_u8m2( \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m2_u8m1(idx, 0), \ - __riscv_vsetvlmax_e8m1()), \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m2_u8m1(idx, 1), \ - __riscv_vsetvlmax_e8m1())) - -#define simdutf_vrgather_u8m1x4(tbl, idx) \ - __riscv_vcreate_v_u8m1_u8m4( \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m4_u8m1(idx, 0), \ - __riscv_vsetvlmax_e8m1()), \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m4_u8m1(idx, 1), \ - __riscv_vsetvlmax_e8m1()), \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m4_u8m1(idx, 2), \ - __riscv_vsetvlmax_e8m1()), \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m4_u8m1(idx, 3), \ - __riscv_vsetvlmax_e8m1())) - -#if __riscv_zbc -#include -#endif - -#endif // SIMDJSON_RVV_VLS_INTRINSICS_H -/* end file simdjson/rvv-vls/intrinsics.h */ -/* including simdjson/rvv-vls/bitmanipulation.h: #include "simdjson/rvv-vls/bitmanipulation.h" */ -/* begin file simdjson/rvv-vls/bitmanipulation.h */ -#ifndef SIMDJSON_RVV_VLS_BITMANIPULATION_H -#define SIMDJSON_RVV_VLS_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { - return __builtin_ctzll(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { - return __builtin_clzll(input_num); -} - -simdjson_inline long long int count_ones(uint64_t input_num) { - return __builtin_popcountll(input_num); -} - - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_BITMANIPULATION_H -/* end file simdjson/rvv-vls/bitmanipulation.h */ -/* including simdjson/rvv-vls/bitmask.h: #include "simdjson/rvv-vls/bitmask.h" */ -/* begin file simdjson/rvv-vls/bitmask.h */ -#ifndef SIMDJSON_RVV_VLS_BITMASK_H -#define SIMDJSON_RVV_VLS_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { -#if __riscv_zbc - return __riscv_clmul_64(bitmask, ~(uint64_t)0); -#elif __riscv_zvbc - return __riscv_vmv_x(__riscv_vclmul(__riscv_vmv_s_x_u64m1(bitmask, 1), ~(uint64_t)0, 1)); -#else - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; -#endif - return bitmask; -} - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_BITMASK_H - -/* end file simdjson/rvv-vls/bitmask.h */ -/* including simdjson/rvv-vls/simd.h: #include "simdjson/rvv-vls/simd.h" */ -/* begin file simdjson/rvv-vls/simd.h */ -#ifndef SIMDJSON_RVV_VLS_SIMD_H -#define SIMDJSON_RVV_VLS_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace simd { - -#if __riscv_v_fixed_vlen >= 512 - static constexpr size_t VL8 = 512/8; - using vint8_t = vint8m1_t __attribute__((riscv_rvv_vector_bits(512))); - using vuint8_t = vuint8m1_t __attribute__((riscv_rvv_vector_bits(512))); - using vbool_t = vbool8_t __attribute__((riscv_rvv_vector_bits(512/8))); - using vbitmask_t = uint64_t; -#else - static constexpr size_t VL8 = __riscv_v_fixed_vlen/8; - using vint8_t = vint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen))); - using vuint8_t = vuint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen))); - using vbool_t = vbool8_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen/8))); - #if __riscv_v_fixed_vlen == 128 - using vbitmask_t = uint16_t; - #elif __riscv_v_fixed_vlen == 256 - using vbitmask_t = uint32_t; - #endif -#endif - -#if __riscv_v_fixed_vlen == 128 - using vuint8x64_t = vuint8m4_t __attribute__((riscv_rvv_vector_bits(512))); - using vboolx64_t = vbool2_t __attribute__((riscv_rvv_vector_bits(512/8))); -#elif __riscv_v_fixed_vlen == 256 - using vuint8x64_t = vuint8m2_t __attribute__((riscv_rvv_vector_bits(512))); - using vboolx64_t = vbool4_t __attribute__((riscv_rvv_vector_bits(512/8))); -#else - using vuint8x64_t = vuint8m1_t __attribute__((riscv_rvv_vector_bits(512))); - using vboolx64_t = vbool8_t __attribute__((riscv_rvv_vector_bits(512/8))); -#endif - - template - struct simd8; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8 { - vbool_t value; - using bitmask_t = vbitmask_t; - static constexpr int SIZE = sizeof(value); - - simdjson_inline simd8(const vbool_t _value) : value(_value) {} - simdjson_inline simd8() : simd8(__riscv_vmclr_m_b8(VL8)) {} - simdjson_inline simd8(bool _value) : simd8(splat(_value)) {} - - simdjson_inline operator const vbool_t&() const { return value; } - simdjson_inline operator vbool_t&() { return value; } - - static simdjson_inline simd8 splat(bool _value) { - return __riscv_vreinterpret_b8(__riscv_vmv_v_x_u64m1(((uint64_t)!_value)-1, 1)); - } - - simdjson_inline vbitmask_t to_bitmask() const { -#if __riscv_v_fixed_vlen == 128 - return __riscv_vmv_x(__riscv_vreinterpret_u16m1(value)); -#elif __riscv_v_fixed_vlen == 256 - return __riscv_vmv_x(__riscv_vreinterpret_u32m1(value)); -#else - return __riscv_vmv_x(__riscv_vreinterpret_u64m1(value)); -#endif - } - - // Bit operations - simdjson_inline simd8 operator|(const simd8 other) const { return __riscv_vmor(*this, other, VL8); } - simdjson_inline simd8 operator&(const simd8 other) const { return __riscv_vmand(*this, other, VL8); } - simdjson_inline simd8 operator^(const simd8 other) const { return __riscv_vmxor(*this, other, VL8); } - simdjson_inline simd8 bit_andnot(const simd8 other) const { return __riscv_vmandn(other, *this, VL8); } - simdjson_inline simd8 operator~() const { return __riscv_vmnot(*this, VL8); } - simdjson_inline simd8& operator|=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline simd8& operator&=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline simd8& operator^=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Unsigned bytes - template<> - struct simd8 { - - vuint8_t value; - static constexpr int SIZE = sizeof(value); - - simdjson_inline simd8(const vuint8_t _value) : value(_value) {} - simdjson_inline simd8() : simd8(zero()) {} - simdjson_inline simd8(const uint8_t values[VL8]) : simd8(load(values)) {} - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - simdjson_inline simd8(simd8 mask) : value(__riscv_vmerge_vxm_u8m1(zero(), -1, (vbool_t)mask, VL8)) {} - - simdjson_inline operator const vuint8_t&() const { return this->value; } - simdjson_inline operator vuint8_t&() { return this->value; } - - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(vuint8_t{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - static simdjson_inline vuint8_t splat(uint8_t _value) { return __riscv_vmv_v_x_u8m1(_value, VL8); } - static simdjson_inline vuint8_t zero() { return splat(0); } - static simdjson_inline vuint8_t load(const uint8_t values[VL8]) { return __riscv_vle8_v_u8m1(values, VL8); } - - // Bit operations - simdjson_inline simd8 operator|(const simd8 other) const { return __riscv_vor_vv_u8m1( value, other, VL8); } - simdjson_inline simd8 operator&(const simd8 other) const { return __riscv_vand_vv_u8m1( value, other, VL8); } - simdjson_inline simd8 operator^(const simd8 other) const { return __riscv_vxor_vv_u8m1( value, other, VL8); } - simdjson_inline simd8 operator~() const { return __riscv_vnot_v_u8m1(value, VL8); } -#if __riscv_zvbb - simdjson_inline simd8 bit_andnot(const simd8 other) const { return __riscv_vandn_vv_u8m1(other, value, VL8); } -#else - simdjson_inline simd8 bit_andnot(const simd8 other) const { return other & ~*this; } -#endif - simdjson_inline simd8& operator|=(const simd8 other) { value = *this | other; return *this; } - simdjson_inline simd8& operator&=(const simd8 other) { value = *this & other; return *this; } - simdjson_inline simd8& operator^=(const simd8 other) { value = *this ^ other; return *this; } - - simdjson_inline simd8 operator==(const simd8 other) const { return __riscv_vmseq(value, other, VL8); } - simdjson_inline simd8 operator==(uint8_t other) const { return __riscv_vmseq(value, other, VL8); } - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return __riscv_vslideup(__riscv_vslidedown(prev_chunk, VL8-N, VL8), value, N, VL8); - } - - // Store to array - simdjson_inline void store(uint8_t dst[VL8]) const { return __riscv_vse8(dst, value, VL8); } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return __riscv_vsaddu(value, other, VL8); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return __riscv_vssubu(value, other, VL8); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return __riscv_vadd(value, other, VL8); } - simdjson_inline simd8 operator-(const simd8 other) const { return __riscv_vsub(value, other, VL8); } - simdjson_inline simd8& operator+=(const simd8 other) { value = *this + other; return *this; } - simdjson_inline simd8& operator-=(const simd8 other) { value = *this - other; return *this; } - - // Order-specific operations - simdjson_inline simd8 operator<=(const simd8 other) const { return __riscv_vmsleu(value, other, VL8); } - simdjson_inline simd8 operator>=(const simd8 other) const { return __riscv_vmsgeu(value, other, VL8); } - simdjson_inline simd8 operator<(const simd8 other) const { return __riscv_vmsltu(value, other, VL8); } - simdjson_inline simd8 operator>(const simd8 other) const { return __riscv_vmsgtu(value, other, VL8); } - - // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. - simdjson_inline simd8 gt_bits(const simd8 other) const { return simd8(*this > other); } - // Same as <, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. - simdjson_inline simd8 lt_bits(const simd8 other) const { return simd8(*this < other); } - - // Bit-specific operations - simdjson_inline bool any_bits_set_anywhere() const { - return __riscv_vfirst(__riscv_vmsne(value, 0, VL8), VL8) >= 0; - } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } - template - simdjson_inline simd8 shr() const { return __riscv_vsrl(value, N, VL8); } - template - simdjson_inline simd8 shl() const { return __riscv_vsll(value, N, VL8); } - - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return __riscv_vrgather(lookup_table, value, VL8); - } - - // compress inactive elements, to match AVX-512 behavior - template - simdjson_inline void compress(vbitmask_t mask, L * output) const { - mask = (vbitmask_t)~mask; -#if __riscv_v_fixed_vlen == 128 - vbool8_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u16m1(mask, 1)); -#elif __riscv_v_fixed_vlen == 256 - vbool8_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u32m1(mask, 1)); -#else - vbool8_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u64m1(mask, 1)); -#endif - __riscv_vse8_v_u8m1(output, __riscv_vcompress(value, m, VL8), count_ones(mask)); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 { - vint8_t value; - static constexpr int SIZE = sizeof(value); - - simdjson_inline simd8(const vint8_t _value) : value(_value) {} - simdjson_inline simd8() : simd8(zero()) {} - simdjson_inline simd8(const int8_t values[VL8]) : simd8(load(values)) {} - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - - simdjson_inline operator const vint8_t&() const { return this->value; } - simdjson_inline operator vint8_t&() { return this->value; } - - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(vint8_t{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - static simdjson_inline vint8_t splat(int8_t _value) { return __riscv_vmv_v_x_i8m1(_value, VL8); } - static simdjson_inline vint8_t zero() { return splat(0); } - static simdjson_inline vint8_t load(const int8_t values[VL8]) { return __riscv_vle8_v_i8m1(values, VL8); } - - - simdjson_inline void store(int8_t dst[VL8]) const { return __riscv_vse8(dst, value, VL8); } - - // Explicit conversion to/from unsigned - simdjson_inline explicit simd8(const vuint8_t other): simd8(__riscv_vreinterpret_i8m1(other)) {} - simdjson_inline explicit operator simd8() const { return __riscv_vreinterpret_u8m1(value); } - - // Math - simdjson_inline simd8 operator+(const simd8 other) const { return __riscv_vadd(value, other, VL8); } - simdjson_inline simd8 operator-(const simd8 other) const { return __riscv_vsub(value, other, VL8); } - simdjson_inline simd8& operator+=(const simd8 other) { value = *this + other; return *this; } - simdjson_inline simd8& operator-=(const simd8 other) { value = *this - other; return *this; } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val( const simd8 other) const { return __riscv_vmax( value, other, VL8); } - simdjson_inline simd8 min_val( const simd8 other) const { return __riscv_vmin( value, other, VL8); } - simdjson_inline simd8 operator>( const simd8 other) const { return __riscv_vmsgt(value, other, VL8); } - simdjson_inline simd8 operator<( const simd8 other) const { return __riscv_vmslt(value, other, VL8); } - simdjson_inline simd8 operator==(const simd8 other) const { return __riscv_vmseq(value, other, VL8); } - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return __riscv_vslideup(__riscv_vslidedown(prev_chunk, VL8-N, VL8), value, N, VL8); - } - - // Perform a lookup assuming no value is larger than 16 - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return __riscv_vrgather(lookup_table, value, VL8); - } - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - template - struct simd8x64; - template<> - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - vuint8x64_t value; - -#if __riscv_v_fixed_vlen >= 512 - template simd8 get() const { return value; } -#else - template simd8 get() const { return __riscv_vget_u8m1(value, idx); } -#endif - - simdjson_inline operator const vuint8x64_t&() const { return this->value; } - simdjson_inline operator vuint8x64_t&() { return this->value; } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - -#if __riscv_v_fixed_vlen == 128 - simdjson_inline simd8x64(const uint8_t *ptr, size_t n = 64) : value(__riscv_vle8_v_u8m4(ptr, n)) {} -#elif __riscv_v_fixed_vlen == 256 - simdjson_inline simd8x64(const uint8_t *ptr, size_t n = 64) : value(__riscv_vle8_v_u8m2(ptr, n)) {} -#else - simdjson_inline simd8x64(const uint8_t *ptr, size_t n = 64) : value(__riscv_vle8_v_u8m1(ptr, n)) {} -#endif - - simdjson_inline void store(uint8_t ptr[64]) const { - __riscv_vse8(ptr, value, 64); - } - - simdjson_inline bool is_ascii() const { -#if __riscv_v_fixed_vlen == 128 - return __riscv_vfirst(__riscv_vmslt(__riscv_vreinterpret_i8m4(value), 0, 64), 64) < 0; -#elif __riscv_v_fixed_vlen == 256 - return __riscv_vfirst(__riscv_vmslt(__riscv_vreinterpret_i8m2(value), 0, 64), 64) < 0; -#else - return __riscv_vfirst(__riscv_vmslt(__riscv_vreinterpret_i8m1(value), 0, 64), 64) < 0; -#endif - } - - // compress inactive elements, to match AVX-512 behavior - simdjson_inline uint64_t compress(uint64_t mask, uint8_t * output) const { - mask = ~mask; -#if __riscv_v_fixed_vlen == 128 - vboolx64_t m = __riscv_vreinterpret_b2(__riscv_vmv_s_x_u64m1(mask, 1)); -#elif __riscv_v_fixed_vlen == 256 - vboolx64_t m = __riscv_vreinterpret_b4(__riscv_vmv_s_x_u64m1(mask, 1)); -#else - vboolx64_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u64m1(mask, 1)); -#endif - size_t cnt = count_ones(mask); - __riscv_vse8(output, __riscv_vcompress(value, m, 64), cnt); - return cnt; - } - - simdjson_inline uint64_t eq(const uint8_t m) const { - return __riscv_vmv_x(__riscv_vreinterpret_u64m1(__riscv_vmseq(value, m, 64))); - } - - simdjson_inline uint64_t lteq(const uint8_t m) const { - return __riscv_vmv_x(__riscv_vreinterpret_u64m1(__riscv_vmsleu(value, m, 64))); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_SIMD_H -/* end file simdjson/rvv-vls/simd.h */ -/* including simdjson/rvv-vls/stringparsing_defs.h: #include "simdjson/rvv-vls/stringparsing_defs.h" */ -/* begin file simdjson/rvv-vls/stringparsing_defs.h */ -#ifndef SIMDJSON_RVV_VLS_STRINGPARSING_DEFS_H -#define SIMDJSON_RVV_VLS_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint64_t BYTES_PROCESSED = sizeof(simd8); - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint64_t bs_bits; - uint64_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - v.store(dst); - return { (v == '\\').to_bitmask(), (v == '"').to_bitmask() }; -} - - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_STRINGPARSING_DEFS_H -/* end file simdjson/rvv-vls/stringparsing_defs.h */ -/* including simdjson/rvv-vls/numberparsing_defs.h: #include "simdjson/rvv-vls/numberparsing_defs.h" */ -/* begin file simdjson/rvv-vls/numberparsing_defs.h */ -#ifndef SIMDJSON_RVV_VLS_NUMBERPARSING_DEFS_H -#define SIMDJSON_RVV_VLS_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#ifdef JSON_TEST_NUMBERS // for unit testing -void found_invalid_number(const uint8_t *buf); -void found_integer(int64_t result, const uint8_t *buf); -void found_unsigned_integer(uint64_t result, const uint8_t *buf); -void found_float(double result, const uint8_t *buf); -#endif - -namespace simdjson { -namespace rvv_vls { -namespace numberparsing { - -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const char *chars) { - uint64_t val; -#if __riscv_misaligned_fast - memcpy(&val, chars, sizeof(uint64_t)); -#else - val = __riscv_vmv_x(__riscv_vreinterpret_u64m1(__riscv_vlmul_ext_u8m1(__riscv_vle8_v_u8mf2((uint8_t*)chars, 8)))); -#endif - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - return parse_eight_digits_unrolled(reinterpret_cast(chars)); -} - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); - return answer; -} - -} // namespace numberparsing -} // namespace rvv_vls -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_RVV_VLS_NUMBERPARSING_DEFS_H -/* end file simdjson/rvv-vls/numberparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 -/* end file simdjson/rvv-vls/begin.h */ -/* including simdjson/rvv-vls/simd.h: #include */ -/* begin file simdjson/rvv-vls/simd.h */ -#ifndef SIMDJSON_RVV_VLS_SIMD_H -#define SIMDJSON_RVV_VLS_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace simd { - -#if __riscv_v_fixed_vlen >= 512 - static constexpr size_t VL8 = 512/8; - using vint8_t = vint8m1_t __attribute__((riscv_rvv_vector_bits(512))); - using vuint8_t = vuint8m1_t __attribute__((riscv_rvv_vector_bits(512))); - using vbool_t = vbool8_t __attribute__((riscv_rvv_vector_bits(512/8))); - using vbitmask_t = uint64_t; -#else - static constexpr size_t VL8 = __riscv_v_fixed_vlen/8; - using vint8_t = vint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen))); - using vuint8_t = vuint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen))); - using vbool_t = vbool8_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen/8))); - #if __riscv_v_fixed_vlen == 128 - using vbitmask_t = uint16_t; - #elif __riscv_v_fixed_vlen == 256 - using vbitmask_t = uint32_t; - #endif -#endif - -#if __riscv_v_fixed_vlen == 128 - using vuint8x64_t = vuint8m4_t __attribute__((riscv_rvv_vector_bits(512))); - using vboolx64_t = vbool2_t __attribute__((riscv_rvv_vector_bits(512/8))); -#elif __riscv_v_fixed_vlen == 256 - using vuint8x64_t = vuint8m2_t __attribute__((riscv_rvv_vector_bits(512))); - using vboolx64_t = vbool4_t __attribute__((riscv_rvv_vector_bits(512/8))); -#else - using vuint8x64_t = vuint8m1_t __attribute__((riscv_rvv_vector_bits(512))); - using vboolx64_t = vbool8_t __attribute__((riscv_rvv_vector_bits(512/8))); -#endif - - template - struct simd8; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8 { - vbool_t value; - using bitmask_t = vbitmask_t; - static constexpr int SIZE = sizeof(value); - - simdjson_inline simd8(const vbool_t _value) : value(_value) {} - simdjson_inline simd8() : simd8(__riscv_vmclr_m_b8(VL8)) {} - simdjson_inline simd8(bool _value) : simd8(splat(_value)) {} - - simdjson_inline operator const vbool_t&() const { return value; } - simdjson_inline operator vbool_t&() { return value; } - - static simdjson_inline simd8 splat(bool _value) { - return __riscv_vreinterpret_b8(__riscv_vmv_v_x_u64m1(((uint64_t)!_value)-1, 1)); - } - - simdjson_inline vbitmask_t to_bitmask() const { -#if __riscv_v_fixed_vlen == 128 - return __riscv_vmv_x(__riscv_vreinterpret_u16m1(value)); -#elif __riscv_v_fixed_vlen == 256 - return __riscv_vmv_x(__riscv_vreinterpret_u32m1(value)); -#else - return __riscv_vmv_x(__riscv_vreinterpret_u64m1(value)); -#endif - } - - // Bit operations - simdjson_inline simd8 operator|(const simd8 other) const { return __riscv_vmor(*this, other, VL8); } - simdjson_inline simd8 operator&(const simd8 other) const { return __riscv_vmand(*this, other, VL8); } - simdjson_inline simd8 operator^(const simd8 other) const { return __riscv_vmxor(*this, other, VL8); } - simdjson_inline simd8 bit_andnot(const simd8 other) const { return __riscv_vmandn(other, *this, VL8); } - simdjson_inline simd8 operator~() const { return __riscv_vmnot(*this, VL8); } - simdjson_inline simd8& operator|=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline simd8& operator&=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline simd8& operator^=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Unsigned bytes - template<> - struct simd8 { - - vuint8_t value; - static constexpr int SIZE = sizeof(value); - - simdjson_inline simd8(const vuint8_t _value) : value(_value) {} - simdjson_inline simd8() : simd8(zero()) {} - simdjson_inline simd8(const uint8_t values[VL8]) : simd8(load(values)) {} - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - simdjson_inline simd8(simd8 mask) : value(__riscv_vmerge_vxm_u8m1(zero(), -1, (vbool_t)mask, VL8)) {} - - simdjson_inline operator const vuint8_t&() const { return this->value; } - simdjson_inline operator vuint8_t&() { return this->value; } - - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(vuint8_t{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - static simdjson_inline vuint8_t splat(uint8_t _value) { return __riscv_vmv_v_x_u8m1(_value, VL8); } - static simdjson_inline vuint8_t zero() { return splat(0); } - static simdjson_inline vuint8_t load(const uint8_t values[VL8]) { return __riscv_vle8_v_u8m1(values, VL8); } - - // Bit operations - simdjson_inline simd8 operator|(const simd8 other) const { return __riscv_vor_vv_u8m1( value, other, VL8); } - simdjson_inline simd8 operator&(const simd8 other) const { return __riscv_vand_vv_u8m1( value, other, VL8); } - simdjson_inline simd8 operator^(const simd8 other) const { return __riscv_vxor_vv_u8m1( value, other, VL8); } - simdjson_inline simd8 operator~() const { return __riscv_vnot_v_u8m1(value, VL8); } -#if __riscv_zvbb - simdjson_inline simd8 bit_andnot(const simd8 other) const { return __riscv_vandn_vv_u8m1(other, value, VL8); } -#else - simdjson_inline simd8 bit_andnot(const simd8 other) const { return other & ~*this; } -#endif - simdjson_inline simd8& operator|=(const simd8 other) { value = *this | other; return *this; } - simdjson_inline simd8& operator&=(const simd8 other) { value = *this & other; return *this; } - simdjson_inline simd8& operator^=(const simd8 other) { value = *this ^ other; return *this; } - - simdjson_inline simd8 operator==(const simd8 other) const { return __riscv_vmseq(value, other, VL8); } - simdjson_inline simd8 operator==(uint8_t other) const { return __riscv_vmseq(value, other, VL8); } - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return __riscv_vslideup(__riscv_vslidedown(prev_chunk, VL8-N, VL8), value, N, VL8); - } - - // Store to array - simdjson_inline void store(uint8_t dst[VL8]) const { return __riscv_vse8(dst, value, VL8); } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return __riscv_vsaddu(value, other, VL8); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return __riscv_vssubu(value, other, VL8); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return __riscv_vadd(value, other, VL8); } - simdjson_inline simd8 operator-(const simd8 other) const { return __riscv_vsub(value, other, VL8); } - simdjson_inline simd8& operator+=(const simd8 other) { value = *this + other; return *this; } - simdjson_inline simd8& operator-=(const simd8 other) { value = *this - other; return *this; } - - // Order-specific operations - simdjson_inline simd8 operator<=(const simd8 other) const { return __riscv_vmsleu(value, other, VL8); } - simdjson_inline simd8 operator>=(const simd8 other) const { return __riscv_vmsgeu(value, other, VL8); } - simdjson_inline simd8 operator<(const simd8 other) const { return __riscv_vmsltu(value, other, VL8); } - simdjson_inline simd8 operator>(const simd8 other) const { return __riscv_vmsgtu(value, other, VL8); } - - // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. - simdjson_inline simd8 gt_bits(const simd8 other) const { return simd8(*this > other); } - // Same as <, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. - simdjson_inline simd8 lt_bits(const simd8 other) const { return simd8(*this < other); } - - // Bit-specific operations - simdjson_inline bool any_bits_set_anywhere() const { - return __riscv_vfirst(__riscv_vmsne(value, 0, VL8), VL8) >= 0; - } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } - template - simdjson_inline simd8 shr() const { return __riscv_vsrl(value, N, VL8); } - template - simdjson_inline simd8 shl() const { return __riscv_vsll(value, N, VL8); } - - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return __riscv_vrgather(lookup_table, value, VL8); - } - - // compress inactive elements, to match AVX-512 behavior - template - simdjson_inline void compress(vbitmask_t mask, L * output) const { - mask = (vbitmask_t)~mask; -#if __riscv_v_fixed_vlen == 128 - vbool8_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u16m1(mask, 1)); -#elif __riscv_v_fixed_vlen == 256 - vbool8_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u32m1(mask, 1)); -#else - vbool8_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u64m1(mask, 1)); -#endif - __riscv_vse8_v_u8m1(output, __riscv_vcompress(value, m, VL8), count_ones(mask)); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 { - vint8_t value; - static constexpr int SIZE = sizeof(value); - - simdjson_inline simd8(const vint8_t _value) : value(_value) {} - simdjson_inline simd8() : simd8(zero()) {} - simdjson_inline simd8(const int8_t values[VL8]) : simd8(load(values)) {} - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - - simdjson_inline operator const vint8_t&() const { return this->value; } - simdjson_inline operator vint8_t&() { return this->value; } - - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(vint8_t{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - static simdjson_inline vint8_t splat(int8_t _value) { return __riscv_vmv_v_x_i8m1(_value, VL8); } - static simdjson_inline vint8_t zero() { return splat(0); } - static simdjson_inline vint8_t load(const int8_t values[VL8]) { return __riscv_vle8_v_i8m1(values, VL8); } - - - simdjson_inline void store(int8_t dst[VL8]) const { return __riscv_vse8(dst, value, VL8); } - - // Explicit conversion to/from unsigned - simdjson_inline explicit simd8(const vuint8_t other): simd8(__riscv_vreinterpret_i8m1(other)) {} - simdjson_inline explicit operator simd8() const { return __riscv_vreinterpret_u8m1(value); } - - // Math - simdjson_inline simd8 operator+(const simd8 other) const { return __riscv_vadd(value, other, VL8); } - simdjson_inline simd8 operator-(const simd8 other) const { return __riscv_vsub(value, other, VL8); } - simdjson_inline simd8& operator+=(const simd8 other) { value = *this + other; return *this; } - simdjson_inline simd8& operator-=(const simd8 other) { value = *this - other; return *this; } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val( const simd8 other) const { return __riscv_vmax( value, other, VL8); } - simdjson_inline simd8 min_val( const simd8 other) const { return __riscv_vmin( value, other, VL8); } - simdjson_inline simd8 operator>( const simd8 other) const { return __riscv_vmsgt(value, other, VL8); } - simdjson_inline simd8 operator<( const simd8 other) const { return __riscv_vmslt(value, other, VL8); } - simdjson_inline simd8 operator==(const simd8 other) const { return __riscv_vmseq(value, other, VL8); } - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return __riscv_vslideup(__riscv_vslidedown(prev_chunk, VL8-N, VL8), value, N, VL8); - } - - // Perform a lookup assuming no value is larger than 16 - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return __riscv_vrgather(lookup_table, value, VL8); - } - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - template - struct simd8x64; - template<> - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - vuint8x64_t value; - -#if __riscv_v_fixed_vlen >= 512 - template simd8 get() const { return value; } -#else - template simd8 get() const { return __riscv_vget_u8m1(value, idx); } -#endif - - simdjson_inline operator const vuint8x64_t&() const { return this->value; } - simdjson_inline operator vuint8x64_t&() { return this->value; } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - -#if __riscv_v_fixed_vlen == 128 - simdjson_inline simd8x64(const uint8_t *ptr, size_t n = 64) : value(__riscv_vle8_v_u8m4(ptr, n)) {} -#elif __riscv_v_fixed_vlen == 256 - simdjson_inline simd8x64(const uint8_t *ptr, size_t n = 64) : value(__riscv_vle8_v_u8m2(ptr, n)) {} -#else - simdjson_inline simd8x64(const uint8_t *ptr, size_t n = 64) : value(__riscv_vle8_v_u8m1(ptr, n)) {} -#endif - - simdjson_inline void store(uint8_t ptr[64]) const { - __riscv_vse8(ptr, value, 64); - } - - simdjson_inline bool is_ascii() const { -#if __riscv_v_fixed_vlen == 128 - return __riscv_vfirst(__riscv_vmslt(__riscv_vreinterpret_i8m4(value), 0, 64), 64) < 0; -#elif __riscv_v_fixed_vlen == 256 - return __riscv_vfirst(__riscv_vmslt(__riscv_vreinterpret_i8m2(value), 0, 64), 64) < 0; -#else - return __riscv_vfirst(__riscv_vmslt(__riscv_vreinterpret_i8m1(value), 0, 64), 64) < 0; -#endif - } - - // compress inactive elements, to match AVX-512 behavior - simdjson_inline uint64_t compress(uint64_t mask, uint8_t * output) const { - mask = ~mask; -#if __riscv_v_fixed_vlen == 128 - vboolx64_t m = __riscv_vreinterpret_b2(__riscv_vmv_s_x_u64m1(mask, 1)); -#elif __riscv_v_fixed_vlen == 256 - vboolx64_t m = __riscv_vreinterpret_b4(__riscv_vmv_s_x_u64m1(mask, 1)); -#else - vboolx64_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u64m1(mask, 1)); -#endif - size_t cnt = count_ones(mask); - __riscv_vse8(output, __riscv_vcompress(value, m, 64), cnt); - return cnt; - } - - simdjson_inline uint64_t eq(const uint8_t m) const { - return __riscv_vmv_x(__riscv_vreinterpret_u64m1(__riscv_vmseq(value, m, 64))); - } - - simdjson_inline uint64_t lteq(const uint8_t m) const { - return __riscv_vmv_x(__riscv_vreinterpret_u64m1(__riscv_vmsleu(value, m, 64))); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_SIMD_H -/* end file simdjson/rvv-vls/simd.h */ -/* including generic/amalgamated.h for rvv_vls: #include */ -/* begin file generic/amalgamated.h for rvv_vls */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) -#error generic/dependencies.h must be included before generic/amalgamated.h! -#endif - -/* including generic/base.h for rvv_vls: #include */ -/* begin file generic/base.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { - -struct json_character_block; - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_BASE_H -/* end file generic/base.h for rvv_vls */ -/* including generic/dom_parser_implementation.h for rvv_vls: #include */ -/* begin file generic/dom_parser_implementation.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// Interface a dom parser implementation must fulfill -namespace simdjson { -namespace rvv_vls { -namespace { - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); -simdjson_inline bool is_ascii(const simd8x64& input); - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file generic/dom_parser_implementation.h for rvv_vls */ -/* including generic/json_character_block.h for rvv_vls: #include */ -/* begin file generic/json_character_block.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { - -struct json_character_block { - static simdjson_inline json_character_block classify(const simd::simd8x64& in); - - simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_inline uint64_t op() const noexcept { return _op; } - simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } - - uint64_t _whitespace; - uint64_t _op; -}; - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H -/* end file generic/json_character_block.h for rvv_vls */ -/* end file generic/amalgamated.h for rvv_vls */ -/* including generic/stage1/amalgamated.h for rvv_vls: #include */ -/* begin file generic/stage1/amalgamated.h for rvv_vls */ -// Stuff other things depend on -/* including generic/stage1/base.h for rvv_vls: #include */ -/* begin file generic/stage1/base.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace stage1 { - -class bit_indexer; -template -struct buf_block_reader; -struct json_block; -class json_minifier; -class json_scanner; -struct json_string_block; -class json_string_scanner; -class json_structural_indexer; - -} // namespace stage1 - -namespace utf8_validation { -struct utf8_checker; -} // namespace utf8_validation - -using utf8_validation::utf8_checker; - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H -/* end file generic/stage1/base.h for rvv_vls */ -/* including generic/stage1/buf_block_reader.h for rvv_vls: #include */ -/* begin file generic/stage1/buf_block_reader.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace stage1 { - -// Walks through a buffer in block-sized increments, loading the last part with spaces -template -struct buf_block_reader { -public: - simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t *full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t *dst) const; - /** - * Like get_remainder, for input followed by SIMDJSON_PADDING readable bytes. - * - * get_remainder writes the block with a memset and a memcpy of the remaining - * length, and the vector loads that read it back cannot be forwarded from - * those narrower stores: they wait until the stores reach the cache, which - * costs tens of cycles on every document. Here the block is put together in - * vector registers, from loads that may read into the padding, and written - * with full-width stores that the loads can forward from. - */ - simdjson_inline size_t get_remainder_padded(uint8_t *dst) const; - simdjson_inline void advance(); -private: - const uint8_t *buf; - const size_t len; - const size_t lenminusstep; - size_t idx; -}; - -template -simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} - -template -simdjson_inline size_t buf_block_reader::block_index() { return idx; } - -template -simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; -} - -template -simdjson_inline const uint8_t *buf_block_reader::full_block() const { - return &buf[idx]; -} - -template -simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { - if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; -} - -// keep_mask[128 - n + i] is 0xFF for i < n and 0 otherwise. -alignas(64) static const uint8_t keep_mask[256] = { - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF -}; - -template -simdjson_inline size_t buf_block_reader::get_remainder_padded(uint8_t *dst) const { - static_assert(STEP_SIZE <= 128, "keep_mask covers blocks of up to 128 bytes"); - static_assert(sizeof(simd::simd8) <= SIMDJSON_PADDING, "a vector load at the last byte must stay in the padding"); - const size_t remaining = len - idx; - if (remaining == 0) { return 0; } - const simd::simd8 spaces = simd::simd8::splat(0x20); - for (size_t off = 0; off < STEP_SIZE; off += sizeof(simd::simd8)) { - simd::simd8 v = spaces; - if (off < remaining) { - // Starts before len, so it ends within the padding. - const simd::simd8 in = simd::simd8::load(buf + idx + off); - const simd::simd8 keep = simd::simd8::load(keep_mask + 128 - remaining + off); - v = spaces ^ ((in ^ spaces) & keep); - } - v.store(dst + off); - } - return remaining; -} - -template -simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for rvv_vls */ -/* including generic/stage1/json_escape_scanner.h for rvv_vls: #include */ -/* begin file generic/stage1/json_escape_scanner.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace stage1 { - -/** - * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). - */ -struct json_escape_scanner { - /** The actual escape characters (the backslashes themselves). */ - uint64_t next_is_escaped = 0ULL; - - struct escaped_and_escape { - /** - * Mask of escaped characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 0100100010100101000 - * n \ \ n \ \ - * ``` - */ - uint64_t escaped; - /** - * Mask of escape characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 1001000101001010001 - * \ \ \ \ \ \ \ - * ``` - */ - uint64_t escape; - }; - - /** - * Get a mask of both escape and escaped characters (the characters following a backslash). - * - * @param potential_escape A mask of the character that can escape others (but could be - * escaped itself). e.g. block.eq('\\') - */ - simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { - -#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT - if (!backslash) { return {next_escaped_without_backslashes(), 0}; } -#endif - - // | | Mask (shows characters instead of 1's) | Depth | Instructions | - // |--------------------------------|----------------------------------------|-------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | - // | | ` even odd even odd odd` | | | - // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) - // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) - // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) - // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () - // (*) this is not needed until the next iteration - uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); - uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); - uint64_t escape = escape_and_terminal_code & backslash; - this->next_is_escaped = escape >> 63; - return {escaped, escape}; - } - -private: - static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; - - simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { - uint64_t escaped = this->next_is_escaped; - this->next_is_escaped = 0; - return escaped; - } - - /** - * Returns a mask of the next escape characters (masking out escaped backslashes), along with - * any non-backslash escape codes. - * - * \n \\n \\\n \\\\n returns: - * \n \ \ \n \ \ - * 11 100 1011 10100 - * - * You are expected to mask out the first bit yourself if the previous block had a trailing - * escape. - * - * & the result with potential_escape to get just the escape characters. - * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. - */ - static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { - // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: - // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be - // inverted (\\\ would be 010 instead of 101). - // - // ``` - // string: | ____\\\\_\\\\_____ | - // maybe_escaped | ODD | \ \ \ \ | - // even-aligned ^^^ ^^^^ odd-aligned - // ``` - // - // Taking that into account, our basic strategy is: - // - // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for - // odd-aligned runs. - // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the - // odd bits in odd-aligned runs. - // 3. & with backslash to clean up any stray bits. - // runs are set to 0, and then XORing with "odd": - // - // | | Mask (shows characters instead of 1's) | Instructions | - // |--------------------------------|----------------------------------------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | - // | | ` even odd even odd odd` | - // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) - // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) - // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) - // - - // Escaped characters are characters following an escape. - uint64_t maybe_escaped = potential_escape << 1; - - // To distinguish odd from even escape sequences, therefore, we turn on any *starting* - // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) - // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. - // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. - // - All other odd bytes are 1, and even bytes are 0. - uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; - uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; - - // Now we flip all odd bytes back with xor. This: - // - Makes odd runs of backslashes go from 0000 to 1010 - // - Makes even runs of backslashes go from 1111 to 1010 - // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) - // - Resets all other bytes to 0 - return even_series_codes_and_odd_bits ^ ODD_BITS; - } -}; - -} // namespace stage1 -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_escape_scanner.h for rvv_vls */ -/* including generic/stage1/json_string_scanner.h for rvv_vls: #include */ -/* begin file generic/stage1/json_string_scanner.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace stage1 { - -struct json_string_block { - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : - _escaped(escaped), _quote(quote), _in_string(in_string) {} - - // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } - // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } - // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } - // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } - - // escaped characters (backslashed--does not include the hex characters after \u) - uint64_t _escaped; - // real quotes (non-escaped ones) - uint64_t _quote; - // string characters (includes start quote but not end quote) - uint64_t _in_string; -}; - -// Scans blocks for string characters, storing the state necessary to do so -class json_string_scanner { -public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); - -private: - // Scans for escape characters - json_escape_scanner escape_scanner{}; - // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). - uint64_t prev_in_string = 0ULL; -}; - -// -// Return a mask of all string characters plus end quotes. -// -// prev_escaped is overflow saying whether the next character is escaped. -// prev_in_string is overflow saying whether we're still in a string. -// -// Backslash sequences outside of quotes will be detected in stage 2. -// -simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = escape_scanner.next(backslash).escaped; - const uint64_t quote = in.eq('"') & ~escaped; - - // - // prefix_xor flips on bits inside the string (and flips off the end quote). - // - // Then we xor with prev_in_string: if we were in a string already, its effect is flipped - // (characters inside strings are outside, and characters outside strings are inside). - // - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - - // - // Check if we're still in a string at the end of the box so the next block will know - // - prev_in_string = uint64_t(static_cast(in_string) >> 63); - - // Use ^ to turn the beginning quote off, and the end quote on. - - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_string_block(escaped, quote, in_string); -} - -simdjson_really_inline error_code json_string_scanner::finish() { - if (prev_in_string) { - return UNCLOSED_STRING; - } - return SUCCESS; -} - -} // namespace stage1 -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_string_scanner.h for rvv_vls */ -/* including generic/stage1/utf8_lookup4_algorithm.h for rvv_vls: #include */ -/* begin file generic/stage1/utf8_lookup4_algorithm.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace utf8_validation { - -using namespace simd; - - simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { -// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) -// Bit 1 = Too Long (ASCII followed by continuation) -// Bit 2 = Overlong 3-byte -// Bit 4 = Surrogate -// Bit 5 = Overlong 2-byte -// Bit 7 = Two Continuations - constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______ - // 11______ 11______ - constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______ - constexpr const uint8_t OVERLONG_3 = 1<<2; // 11100000 100_____ - constexpr const uint8_t SURROGATE = 1<<4; // 11101101 101_____ - constexpr const uint8_t OVERLONG_2 = 1<<5; // 1100000_ 10______ - constexpr const uint8_t TWO_CONTS = 1<<7; // 10______ 10______ - constexpr const uint8_t TOO_LARGE = 1<<3; // 11110100 1001____ - // 11110100 101_____ - // 11110101 1001____ - // 11110101 101_____ - // 1111011_ 1001____ - // 1111011_ 101_____ - // 11111___ 1001____ - // 11111___ 101_____ - constexpr const uint8_t TOO_LARGE_1000 = 1<<6; - // 11110101 1000____ - // 1111011_ 1000____ - // 11111___ 1000____ - constexpr const uint8_t OVERLONG_4 = 1<<6; // 11110000 1000____ - - const simd8 byte_1_high = prev1.shr<4>().lookup_16( - // 0_______ ________ - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - // 10______ ________ - TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, - // 1100____ ________ - TOO_SHORT | OVERLONG_2, - // 1101____ ________ - TOO_SHORT, - // 1110____ ________ - TOO_SHORT | OVERLONG_3 | SURROGATE, - // 1111____ ________ - TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 - ); - constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( - // ____0000 ________ - CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, - // ____0001 ________ - CARRY | OVERLONG_2, - // ____001_ ________ - CARRY, - CARRY, - - // ____0100 ________ - CARRY | TOO_LARGE, - // ____0101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____011_ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - - // ____1___ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____1101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000 - ); - const simd8 byte_2_high = input.shr<4>().lookup_16( - // ________ 0_______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - - // ________ 1000____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, - // ________ 1001____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, - // ________ 101_____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - - // ________ 11______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT - ); - return (byte_1_high & byte_1_low & byte_2_high); - } - simdjson_inline simd8 check_multibyte_lengths(const simd8 input, - const simd8 prev_input, const simd8 sc) { - simd8 prev2 = input.prev<2>(prev_input); - simd8 prev3 = input.prev<3>(prev_input); - simd8 must23 = must_be_2_3_continuation(prev2, prev3); - simd8 must23_80 = must23 & uint8_t(0x80); - return must23_80 ^ sc; - } - - // - // Return nonzero if there are incomplete multibyte characters at the end of the block: - // e.g. if there is a 4-byte character, but it's 3 bytes from the end. - // - simdjson_inline simd8 is_incomplete(const simd8 input) { - // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): - // ... 1111____ 111_____ 11______ -#if SIMDJSON_IMPLEMENTATION_ICELAKE || (SIMDJSON_IMPLEMENTATION_RVV_VLS && __riscv_v_fixed_vlen >= 512) - static const uint8_t max_array[64] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#else - static const uint8_t max_array[32] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 - }; -#endif - const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); - return input.gt_bits(max_value); - } - - struct utf8_checker { - // If this is nonzero, there has been a UTF-8 error. - simd8 error; - // The last input we received - simd8 prev_input_block; - // Whether the last input we received was incomplete (used for ASCII fast path) - simd8 prev_incomplete; - - // - // Check whether the current bytes are valid UTF-8. - // - simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { - // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes - // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) - simd8 prev1 = input.prev<1>(prev_input); - simd8 sc = check_special_cases(input, prev1); - this->error |= check_multibyte_lengths(input, prev_input, sc); - } - - // The only problem that can happen at EOF is that a multibyte character is too short - // or a byte value too large in the last bytes: check_special_cases only checks for bytes - // too large in the first of two bytes. - simdjson_inline void check_eof() { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; - } - - simdjson_inline void check_next_input(const simd8x64& input) { - if(simdjson_likely(is_ascii(input))) { - this->error |= this->prev_incomplete; - } else { - // you might think that a for-loop would work, but under Visual Studio, it is not good enough. - static_assert((simd8x64::NUM_CHUNKS == 1) - ||(simd8x64::NUM_CHUNKS == 2) - || (simd8x64::NUM_CHUNKS == 4), - "We support one, two or four chunks per 64-byte block."); - SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 1) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 2) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 4) { - this->check_utf8_bytes(input.get<0>(), this->prev_input_block); - this->check_utf8_bytes(input.get<1>(), input.get<0>()); - this->check_utf8_bytes(input.get<2>(), input.get<1>()); - this->check_utf8_bytes(input.get<3>(), input.get<2>()); - } - this->prev_incomplete = is_incomplete(input.get::NUM_CHUNKS-1>()); - this->prev_input_block = input.get::NUM_CHUNKS-1>(); - } - } - // do not forget to call check_eof! - simdjson_warn_unused simdjson_inline error_code errors() { - return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; - } - - }; // struct utf8_checker -} // namespace utf8_validation - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H -/* end file generic/stage1/utf8_lookup4_algorithm.h for rvv_vls */ -/* including generic/stage1/json_scanner.h for rvv_vls: #include */ -/* begin file generic/stage1/json_scanner.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace stage1 { - -/** - * A block of scanned json, with information on operators and scalars. - * - * We seek to identify pseudo-structural characters. Anything that is inside - * a string must be omitted (hence & ~_string.string_tail()). - * Otherwise, pseudo-structural characters come in two forms. - * 1. We have the structural characters ([,],{,},:, comma). The - * term 'structural character' is from the JSON RFC. - * 2. We have the 'scalar pseudo-structural characters'. - * Scalars are quotes, and any character except structural characters and white space. - * - * To identify the scalar pseudo-structural characters, we must look at what comes - * before them: it must be a space, a quote or a structural characters. - * Starting with simdjson v0.3, we identify them by - * negation: we identify everything that is followed by a non-quote scalar, - * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. - */ -struct json_block { -public: - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - - /** - * The start of structurals. - * In simdjson prior to v0.3, these were called the pseudo-structural characters. - **/ - simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } - /** All JSON whitespace (i.e. not in a string) */ - simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } - - // Helpers - - /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } - /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } - - // string and escape characters - json_string_block _string; - // whitespace, structural characters ('operators'), scalars - json_character_block _characters; - // whether the previous character was a scalar - uint64_t _follows_potential_nonquote_scalar; -private: - // Potential structurals (i.e. disregarding strings) - - /** - * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". - * They may reside inside a string. - **/ - simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } - /** - * The start of non-operator runs, like 123, true and "abc". - * It main reside inside a string. - **/ - simdjson_inline uint64_t potential_scalar_start() const noexcept { - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space - // then we know that it is irrelevant structurally. - return _characters.scalar() & ~follows_potential_scalar(); - } - /** - * Whether the given character is immediately after a non-operator like 123, true. - * The characters following a quote are not included. - */ - simdjson_inline uint64_t follows_potential_scalar() const noexcept { - // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character - // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a - // white space. - // It is understood that within quoted region, anything at all could be marked (irrelevant). - return _follows_potential_nonquote_scalar; - } -}; - -/** - * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. - * - * The scanner starts by calculating two distinct things: - * - string characters (taking \" into account) - * - structural characters or 'operators' ([]{},:, comma) - * and scalars (runs of non-operators like 123, true and "abc") - * - * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: - * in particular, the operator/scalar bit will find plenty of things that are actually part of - * strings. When we're done, json_block will fuse the two together by masking out tokens that are - * part of a string. - */ -class json_scanner { -public: - json_scanner() = default; - simdjson_inline json_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_warn_unused simdjson_inline error_code finish(); - -private: - // Whether the last character of the previous iteration is part of a scalar token - // (anything except whitespace or a structural character/'operator'). - uint64_t prev_scalar = 0ULL; - json_string_scanner string_scanner{}; -}; - - -// -// Check if the current character immediately follows a matching character. -// -// For example, this checks for quotes with backslashes in front of them: -// -// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); -// -simdjson_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; -} - -simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { - json_string_block strings = string_scanner.next(in); - // identifies the white-space and the structural characters - json_character_block characters = json_character_block::classify(in); - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). - // - // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) - // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential - // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we - // may need to add an extra check when parsing strings. - // - // Performance: there are many ways to skin this cat. - const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); - uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_block( - strings,// strings is a function-local object so either it moves or the copy is elided. - characters, - follows_nonquote_scalar - ); -} - -simdjson_warn_unused simdjson_inline error_code json_scanner::finish() { - return string_scanner.finish(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H -/* end file generic/stage1/json_scanner.h for rvv_vls */ - -// All other declarations -/* including generic/stage1/find_next_document_index.h for rvv_vls: #include */ -/* begin file generic/stage1/find_next_document_index.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace stage1 { - -/** - * This algorithm is used to quickly identify the last structural position that - * makes up a complete document. - * - * It does this by going backwards and finding the last *document boundary* (a - * place where one value follows another without a comma between them). If the - * last document (the characters after the boundary) has an equal number of - * start and end brackets, it is considered complete. - * - * Simply put, we iterate over the structural characters, starting from - * the end. We consider that we found the end of a JSON document when the - * first element of the pair is NOT one of these characters: '{' '[' ':' ',' - * and when the second element is NOT one of these characters: '}' ']' ':' ','. - * - * This simple comparison works most of the time, but it does not cover cases - * where the batch's structural indexes contain a perfect amount of documents. - * In such a case, we do not have access to the structural index which follows - * the last document, therefore, we do not have access to the second element in - * the pair, and that means we cannot identify the last document. To fix this - * issue, we keep a count of the open and closed curly/square braces we found - * while searching for the pair. When we find a pair AND the count of open and - * closed curly/square braces is the same, we know that we just passed a - * complete document, therefore the last json buffer location is the end of the - * batch. - */ -simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { - const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - const uint8_t e = parser.buf[len - 1]; - return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && - e != ' ' && e != '\t' && e != '\n' && e != '\r'; -} - -/** - * The structural index past the last one of a partial batch is the end of this - * window, not a token of it: the byte there is the first byte of the next - * window. A finished value must not be rejected because that byte is ',', ':', - * ']' or '}'. A last document that itself ends with '{', '[', ',' or ':' is - * invalid JSON, yet its brackets may balance: walking it, the On-Demand API - * could read the next batch as a value. We leave such a document to the next - * batch, so that the batch ends between two documents. - */ -simdjson_inline bool ends_with_unfinished_document(dom_parser_implementation &parser) { - const uint8_t last = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - return last == '{' || last == '[' || last == ',' || last == ':'; -} - -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { - // Variant: do not count separately, just figure out depth - if(parser.n_structural_indexes == 0) { return 0; } - auto arr_cnt = 0; - auto obj_cnt = 0; - for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { - auto idxb = parser.structural_indexes[i]; - switch (parser.buf[idxb]) { - case ':': - case ',': - continue; - case '}': - obj_cnt--; - continue; - case ']': - arr_cnt--; - continue; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - auto idxa = parser.structural_indexes[i - 1]; - switch (parser.buf[idxa]) { - case '{': - case '[': - case ':': - case ',': - continue; - } - // Last document is complete, so the next document will appear after! - if (!arr_cnt && !obj_cnt) { - return defer_last ? i : parser.n_structural_indexes; - } - // Last document is incomplete; mark the document at i + 1 as the next one - return i; - } - // If we made it to the end, we want to finish counting to see if we have a full document. - switch (parser.buf[parser.structural_indexes[0]]) { - case '}': - obj_cnt--; - break; - case ']': - arr_cnt--; - break; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - if (!arr_cnt && !obj_cnt) { - // We have a complete document. - return defer_last ? 0 : parser.n_structural_indexes; - } - return 0; -} - -/** - * Sentinel value returned to indicate a document started but didn't fit - * (CAPACITY error), as opposed to 0 which means no document content found - * (EMPTY). - */ -constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; - -/** - * For RFC 7464 JSON text sequences, filter RS from structural indexes and - * find batch boundaries. - * - * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. - * RS bytes appear in structural_indexes as they are classified as scalars. - * This function: - * 1. Scans structural_indexes to find and count RS positions - * 2. Filters RS out of structural_indexes in-place - * 3. Determines batch boundaries based on RS positions - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @param scan_len Offset past which no value start may be derived. Equals len - * unless stage 1 dropped a trailing unclosed string, whose bytes it - * never validated. - * @return The number of structural indexes to keep (after RS filtering), - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t find_next_document_index_json_sequence( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start, - size_t scan_len) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Phase 1: Scan structural_indexes to find RS positions and handle them. - // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS - // is already in structural_indexes (it's an operator). For scalars like numbers, - // the digit following RS is NOT in structural_indexes because the scanner sees - // RS as a scalar, making the digit a scalar continuation, not a start. - // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the - // actual value start position. - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_rs_pos = 0; - uint32_t rs_count = 0; - - for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { - const uint32_t pos = parser.structural_indexes[read_idx]; - if (parser.buf[pos] == 0x1E) { - // This is an RS character - find the actual JSON value start. - last_rs_pos = pos; - rs_count++; - // Skip past this RS and any whitespace *and any additional RSes* - // to locate the real value. Consecutive RSes are degenerate - // "empty records" per RFC 7464; we collapse them here. They do - // not always appear as separate entries in structural_indexes - // because the scanner groups runs of adjacent non-whitespace - // scalar bytes (including RS) into a single scalar start. - uint32_t value_start = pos + 1; - while (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { - value_start++; - } else if (c == 0x1E) { - // Collapsed empty record. Still count it so rs_count reflects - // the true number of record markers and last_rs_pos tracks - // the final one. - last_rs_pos = value_start; - rs_count++; - value_start++; - } else { - break; - } - } - // If the scanner emitted additional structurals inside the - // whitespace+RS run we just walked over (i.e., isolated RSes - // separated by whitespace), skip past them so we do not - // double-count or double-emit. - while (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] < value_start) { - read_idx++; - } - // Check if the value start is an operator (always present in - // scanner structural_indexes) or a scalar-like start (which may - // be missing from structural_indexes and must be added here). - // Note: '"' is NOT always in structural_indexes. The scanner - // classifies '"' as a scalar character and emits it as a - // structural only when it is a *scalar start* (preceded by - // whitespace or an operator). When '"' immediately follows an - // RS (which the scanner also classifies as scalar), it is - // treated as a scalar continuation and not emitted - so we - // must add it here just like any other scalar value. - if (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - const bool is_operator = - (c == '{' || c == '}' || c == '[' || c == ']' || - c == ':' || c == ','); - // If the next scanner structural is exactly at value_start, - // the scanner already emitted it (it followed whitespace) and - // we must not add a duplicate - a subsequent iteration will - // copy it into write_idx. - const bool already_emitted = - (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] == value_start); - if (!is_operator && !already_emitted) { - // Scalar value (number/true/false/null/string) - add its - // position since scanner missed it. - parser.structural_indexes[write_idx++] = value_start; - } - } - } else { - // Not RS, copy to output - parser.structural_indexes[write_idx++] = pos; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { - // Only RS markers here: the last one opens a record continuing past the - // window, so that is where the next batch resumes. - if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } - return 0; - } - if (rs_count == 0) { - // No RS found; for final batch, try generic boundary detection - return is_final ? find_next_document_index(parser) : 0; - } - - // Phase 2: Determine batch boundaries based on RS positions - - if (is_final) { - // Final batch: all documents are complete (last one ends at EOF). - // In json_sequence mode, RS markers define document boundaries, so all - // remaining structurals form complete documents. Return them all directly. - // (Calling find_next_document_index() would fail for scalar documents.) - return parser.n_structural_indexes; - } - - // Partial batch: need to find complete documents only. - // A document starting at an RS is complete if there is another RS after it. - next_batch_start = last_rs_pos; - - if (rs_count < 2) { - // Only one RS, so we have at most one document that may be incomplete. - // We cannot confirm it is complete without another RS. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least 2 RS markers. The last complete document ends before last_rs_pos. - - // Find the structural index cutoff: keep only structurals < last_rs_pos. - // Since we already filtered RS, all remaining structurals are valid. - // We iterate backward to find the last structural before last_rs_pos. - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_rs_pos) { - keep_count = i; - break; - } - } - - // No structurals before the last RS - no complete documents - if (keep_count == 0) { return 0; } - - // All documents before the last RS are complete by definition (the next RS - // confirms their end). No need to call find_next_document_index() which - // would fail for scalar documents like `1` or `"hello"`. - return keep_count; -} - -/** - * Filter comma-delimited documents by removing root-level commas from - * structural indexes. - * - * For comma-delimited format like `{...},{...},{...}`, we need to remove - * the commas that separate documents (depth 0) while preserving commas - * inside arrays and objects (depth > 0). - * - * After filtering, the structural indexes look like whitespace-delimited - * documents, so find_next_document_index() works unchanged. - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @return The number of structural indexes to keep, - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t filter_comma_delimited( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Track depth to identify root-level commas (depth 0) - int depth = 0; - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_root_comma_pos = 0; - uint32_t root_comma_count = 0; - - for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { - uint32_t idx = parser.structural_indexes[i]; - uint8_t c = parser.buf[idx]; - - switch (c) { - case '{': case '[': - depth++; - parser.structural_indexes[write_idx++] = idx; - break; - case '}': case ']': - depth--; - parser.structural_indexes[write_idx++] = idx; - break; - case ',': - if (depth == 0) { - // Root-level comma = document boundary, skip it - last_root_comma_pos = idx; - root_comma_count++; - continue; - } - parser.structural_indexes[write_idx++] = idx; - break; - default: - // Colons, scalars, etc. - parser.structural_indexes[write_idx++] = idx; - break; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { return 0; } - - if (is_final) { - // Final batch: use standard boundary detection on filtered indexes - return find_next_document_index(parser); - } - - // Partial batch: need to find complete documents only. - // A document ending with a root comma is complete. - if (root_comma_count == 0) { - // No root commas found; we cannot confirm any document is complete. - // The whole batch might be one incomplete document. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least one root comma. Documents before the last comma are complete. - next_batch_start = last_root_comma_pos + 1; - - // Find the structural index cutoff: keep only structurals < last_root_comma_pos - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_root_comma_pos) { - keep_count = i; - break; - } - } - - if (keep_count == 0) { return 0; } - - // Use standard boundary detection on the complete portion - parser.n_structural_indexes = keep_count; - return find_next_document_index(parser); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H -/* end file generic/stage1/find_next_document_index.h for rvv_vls */ -/* including generic/stage1/json_minifier.h for rvv_vls: #include */ -/* begin file generic/stage1/json_minifier.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace stage1 { - -class json_minifier { -public: - template - static error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept; - -private: - simdjson_inline json_minifier(uint8_t *_dst) - : dst{_dst} - {} - template - simdjson_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_warn_unused simdjson_inline error_code finish(uint8_t *dst_start, size_t &dst_len); - json_scanner scanner{}; - uint8_t *dst; -}; - -simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { - uint64_t mask = block.whitespace(); - dst += in.compress(mask, dst); -} - -simdjson_warn_unused simdjson_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { - error_code error = scanner.finish(); - if (error) { dst_len = 0; return error; } - dst_len = dst - dst_start; - return SUCCESS; -} - -template<> -simdjson_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - simd::simd8x64 in_2(block_buf+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1); - this->next(in_2, block_2); - reader.advance(); -} - -template<> -simdjson_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block_buf); - json_block block_1 = scanner.next(in_1); - this->next(block_buf, block_1); - reader.advance(); -} - -template -error_code json_minifier::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) noexcept { - buf_block_reader reader(buf, len); - json_minifier minifier(dst); - - // Index the first n-1 blocks - while (reader.has_full_block()) { - minifier.step(reader.full_block(), reader); - } - - // Index the last (remainder) block, padded with spaces - uint8_t block[STEP_SIZE]; - size_t remaining_bytes = reader.get_remainder(block); - if (remaining_bytes > 0) { - // We do not want to write directly to the output stream. Rather, we write - // to a local buffer (for safety). - uint8_t out_block[STEP_SIZE]; - uint8_t * const guarded_dst{minifier.dst}; - minifier.dst = out_block; - minifier.step(block, reader); - size_t to_write = minifier.dst - out_block; - // In some cases, we could be enticed to consider the padded spaces - // as part of the string. This is fine as long as we do not write more - // than we consumed. - if(to_write > remaining_bytes) { to_write = remaining_bytes; } - memcpy(guarded_dst, out_block, to_write); - minifier.dst = guarded_dst + to_write; - } - return minifier.finish(dst, dst_len); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H -/* end file generic/stage1/json_minifier.h for rvv_vls */ -/* including generic/stage1/json_structural_indexer.h for rvv_vls: #include */ -/* begin file generic/stage1/json_structural_indexer.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace stage1 { - -class bit_indexer { -public: - uint32_t *tail; - - simdjson_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} - -#if SIMDJSON_PREFER_REVERSE_BITS - /** - * ARM lacks a fast trailing zero instruction, but it has a fast - * bit reversal instruction and a fast leading zero instruction. - * Thus it may be profitable to reverse the bits (once) and then - * to rely on a sequence of instructions that call the leading - * zero instruction. - * - * Performance notes: - * The chosen routine is not optimal in terms of data dependency - * since zero_leading_bit might require two instructions. However, - * it tends to minimize the total number of instructions which is - * beneficial. - */ - simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { - int lz = leading_zeroes(rev_bits); - this->tail[i] = static_cast(idx) + lz; - rev_bits = zero_leading_bit(rev_bits, lz); - } -#else - /** - * Under recent x64 systems, we often have both a fast trailing zero - * instruction and a fast 'clear-lower-bit' instruction so the following - * algorithm can be competitive. - */ - - simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { - this->tail[i] = idx + trailing_zeroes(bits); - bits = clear_lowest_bit(bits); - } -#endif // SIMDJSON_PREFER_REVERSE_BITS - - template - simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { - write_index(idx, bits, START); - SIMDJSON_IF_CONSTEXPR (N > 1) { - write_indexes<(N-1>0?START+1:START), (N-1>=0?N-1:1)>(idx, bits); - } - return START+N; - } - - template - simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { - write_indexes(idx, bits); - SIMDJSON_IF_CONSTEXPR ((START+STEP) < END) { - if (simdjson_unlikely((START+STEP) < cnt)) { - write_indexes_stepped<(START+STEP(idx, bits, cnt); - } - } - return ((END-START) % STEP) == 0 ? END : (END-START) - ((END-START) % STEP) + STEP; - } - - // flatten out values in 'bits' assuming that they are are to have values of idx - // plus their position in the bitvector, and store these indexes at - // base_ptr[base] incrementing base as we go - // will potentially store extra values beyond end of valid bits, so base_ptr - // needs to be large enough to handle this - // - // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it - // will provide its own version of the code. -#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - simdjson_inline void write(uint32_t idx, uint64_t bits); -#else - simdjson_inline void write(uint32_t idx, uint64_t bits) { - // In some instances, the next branch is expensive because it is mispredicted. - // Unfortunately, in other cases, - // it helps tremendously. - if (bits == 0) - return; - - int cnt = static_cast(count_ones(bits)); - -#if SIMDJSON_PREFER_REVERSE_BITS - bits = reverse_bits(bits); -#endif -#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP - static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; -#else - static constexpr const int STEP = 4; -#endif - static constexpr const int STEP_UNTIL = 24; - - write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); - SIMDJSON_IF_CONSTEXPR (STEP_UNTIL < 64) { - if (simdjson_unlikely(STEP_UNTIL < cnt)) { - for (int i=STEP_UNTIL; itail += cnt; - } -#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -}; - -class json_structural_indexer { -public: - /** - * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. - * - * @param partial Setting the partial parameter to true allows the find_structural_bits to - * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If - * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. - */ - template - static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept; - -private: - simdjson_inline json_structural_indexer(uint32_t *structural_indexes); - template - simdjson_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_warn_unused simdjson_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); - - json_scanner scanner{}; - utf8_checker checker{}; - bit_indexer indexer; - uint64_t prev_structurals = 0; - uint64_t unescaped_chars_error = 0; -}; - -simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} - -// Skip the last character if it is partial -simdjson_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { - if (simdjson_unlikely(len < 3)) { - switch (len) { - case 2: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left - return len; - case 1: - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - return len; - case 0: - return len; - } - } - if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left - return len; -} - -// -// PERF NOTES: -// We pipe 2 inputs through these stages: -// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load -// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. -// 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. -// The output of step 1 depends entirely on this information. These functions don't quite use -// up enough CPU: the second half of the functions is highly serial, only using 1 execution core -// at a time. The second input's scans has some dependency on the first ones finishing it, but -// they can make a lot of progress before they need that information. -// 3. Step 1 does not use enough capacity, so we run some extra stuff while we're waiting for that -// to finish: utf-8 checks and generating the output from the last iteration. -// -// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all -// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough -// workout. -// -template -error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept { - if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } - // We guard the rest of the code so that we can assume that len > 0 throughout. - if (len == 0) { return EMPTY; } - if (is_streaming(partial)) { - len = trim_partial_utf8(buf, len); - // If you end up with an empty window after trimming - // the partial UTF-8 bytes, then chances are good that you - // have an UTF-8 formatting error. - if(len == 0) { return UTF8_ERROR; } - } - buf_block_reader reader(buf, len); - json_structural_indexer indexer(parser.structural_indexes.get()); - - // Read all but the last block - while (reader.has_full_block()) { - indexer.step(reader.full_block(), reader); - } - // Take care of the last block (will always be there unless file is empty which is - // not supposed to happen.) - uint8_t block[STEP_SIZE]; - const size_t remaining = parser._unpadded ? reader.get_remainder(block) : reader.get_remainder_padded(block); - if (simdjson_unlikely(remaining == 0)) { return UNEXPECTED_ERROR; } - indexer.step(block, reader); - return indexer.finish(parser, reader.block_index(), len, partial); -} - -template<> -simdjson_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { - simd::simd8x64 in_1(block); - simd::simd8x64 in_2(block+64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1, reader.block_index()); - this->next(in_2, block_2, reader.block_index()+64); - reader.advance(); -} - -template<> -simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { - simd::simd8x64 in_1(block); - json_block block_1 = scanner.next(in_1); - this->next(in_1, block_1, reader.block_index()); - reader.advance(); -} - -simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { - uint64_t unescaped = in.lteq(0x1F); -#if SIMDJSON_UTF8VALIDATION - checker.check_next_input(in); -#endif - indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser - prev_structurals = block.structural_start(); - unescaped_chars_error |= block.non_quote_inside_string(unescaped); -} - -// Drops the unclosed string that ends a json_sequence batch and leaves its -// start in structural_indexes[n_structural_indexes]. The last index is its -// opening quote, unless the quote directly follows an RS: the scanner reads -// RS as a scalar byte, so it indexes the RS and not the quote. Keep the RS. -simdjson_inline void drop_json_sequence_unclosed_string(dom_parser_implementation &parser, size_t len) noexcept { - uint32_t quote = parser.structural_indexes[parser.n_structural_indexes - 1]; - if (parser.buf[quote] == '"') { - parser.n_structural_indexes--; - return; - } - while (quote < len && parser.buf[quote] != '"') { quote++; } - parser.structural_indexes[parser.n_structural_indexes] = quote; -} - -simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial) { - // Write out the final iteration's structurals - indexer.write(uint32_t(idx-64), prev_structurals); - error_code error = scanner.finish(); - // We deliberately break down the next expression so that it is - // human readable. - const bool should_we_exit = is_streaming(partial) ? - ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING - : (error != SUCCESS); // if partial is false, we must have SUCCESS - const bool have_unclosed_string = (error == UNCLOSED_STRING); - parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); - /*** - * The On-Demand API requires special padding. - * - * This is related to https://github.com/simdjson/simdjson/issues/906 - * Basically, we want to make sure that if the parsing continues beyond the last (valid) - * structural character, it quickly stops. - * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. - * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing - * continues, then it must be [,] or }. - * Suppose it is ] or }. We backtrack to the first character, what could it be that would - * not trigger an error? It could be ] or } but no, because you can't start a document that way. - * It can't be a comma, a colon or any simple value. So the only way we could continue is - * if the repeated character is [. But if so, the document must start with [. But if the document - * starts with [, it should end with ]. If we enforce that rule, then we would get - * ][[ which is invalid. - * - * This is illustrated with the test array_iterate_unclosed_error() on the following input: - * R"({ "a": [,,)" - **/ - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final - parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); - parser.structural_indexes[parser.n_structural_indexes + 2] = 0; - parser.next_structural_index = 0; - - // Bail out only once the count and sentinels above are set: - // document_stream::truncated_bytes() reads them even on error. - if (simdjson_unlikely(should_we_exit)) { return error; } - if (unescaped_chars_error) { return UNESCAPED_CHARS; } - - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - return EMPTY; - } - if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { - return UNEXPECTED_ERROR; - } - if (partial == stage1_mode::streaming_partial) { - // If we have an unclosed string, then the last structural - // will be the quote and we want to make sure to omit it. - if(have_unclosed_string) { - parser.n_structural_indexes--; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - // We truncate the input to the end of the last complete document (or zero). - const bool partial_scalar = !have_unclosed_string && ends_with_partial_scalar(parser, len); - const bool unfinished = ends_with_unfinished_document(parser); - auto new_structural_indexes = find_next_document_index(parser, partial_scalar || unfinished); - if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { - // The batch holds a single document, whose brackets balance but which - // cannot be valid JSON: it is not too big, it is invalid. - if (unfinished && find_next_document_index(parser, partial_scalar) != 0) { return TAPE_ERROR; } - if(parser.structural_indexes[0] == 0) { - // If the buffer is partial and we started at index 0 but the document is - // incomplete, it's too big to parse. - return CAPACITY; - } else { - // It is possible that the document could be parsed, we just had a lot - // of white space. - parser.n_structural_indexes = 0; - return EMPTY; - } - } - parser.n_structural_indexes = new_structural_indexes; - } else if (partial == stage1_mode::streaming_final) { - if(have_unclosed_string) { parser.n_structural_indexes--; } - // We truncate the input to the end of the last complete document (or zero). - // Because partial == stage1_mode::streaming_final, it means that we may - // silently ignore trailing garbage. Though it sounds bad, we do it - // deliberately because many people who have streams of JSON documents - // will truncate them for processing. E.g., imagine that you are uncompressing - // the data from a size file or receiving it in chunks from the network. You - // may not know where exactly the last document will be. Meanwhile the - // document_stream instances allow people to know the JSON documents they are - // parsing (see the iterator.source() method). - parser.n_structural_indexes = find_next_document_index(parser); - // We store the initial n_structural_indexes so that the client can see - // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, - // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, - // otherwise, it will copy some prior index. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - // This next line is critical, do not change it unless you understand what you are - // doing. - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - // We tolerate an unclosed string at the very end of the stream. Indeed, users - // often load their data in bulk without being careful and they want us to ignore - // the trailing garbage. - return EMPTY; - } - } else if (partial == stage1_mode::json_sequence_partial) { - // RFC 7464: use RS positions for batch boundaries - // A discarded unclosed string also caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - // The next batch starts at an RS, which the On-Demand API never reads as a - // string, a container or a separator. But an invalid last document may make - // it skip that RS as a value: repeat it, so that the index past it is not - // the next record, which can be a string that does not close in this batch. - parser.structural_indexes[parser.n_structural_indexes + 1] = next_batch_start; - } else if (partial == stage1_mode::json_sequence_final) { - // RFC 7464: final batch, last document extends to EOF - // As above: a discarded unclosed string caps how far the filter may scan. - size_t scan_len = len; - if(have_unclosed_string) { - drop_json_sequence_unclosed_string(parser, len); - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); - // The filter compacted structural_indexes in place and restored the EOF - // sentinel past the compacted end, so the copy below is either the start - // of a truncated document or len, as in streaming_final. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } else if (partial == stage1_mode::comma_delimited_partial) { - // Comma-delimited: filter root-level commas, use comma positions for batch boundaries - if(have_unclosed_string) { - parser.n_structural_indexes--; - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - } else if (partial == stage1_mode::comma_delimited_final) { - // Comma-delimited: final batch, last document extends to EOF - if(have_unclosed_string) { parser.n_structural_indexes--; } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } - } - checker.check_eof(); - return checker.errors(); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. -#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H -/* end file generic/stage1/json_structural_indexer.h for rvv_vls */ -/* including generic/stage1/utf8_validator.h for rvv_vls: #include */ -/* begin file generic/stage1/utf8_validator.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace stage1 { - -/** - * Validates that the string is actual UTF-8. - */ -template -bool generic_validate_utf8(const uint8_t * input, size_t length) { - checker c{}; - buf_block_reader<64> reader(input, length); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - c.check_next_input(in); - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - c.check_next_input(in); - reader.advance(); - c.check_eof(); - return c.errors() == error_code::SUCCESS; -} - -bool generic_validate_utf8(const char * input, size_t length) { - return generic_validate_utf8(reinterpret_cast(input),length); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H -/* end file generic/stage1/utf8_validator.h for rvv_vls */ -/* end file generic/stage1/amalgamated.h for rvv_vls */ -/* including generic/stage2/amalgamated.h for rvv_vls: #include */ -/* begin file generic/stage2/amalgamated.h for rvv_vls */ -// Stuff other things depend on -/* including generic/stage2/base.h for rvv_vls: #include */ -/* begin file generic/stage2/base.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace stage2 { - -class json_iterator; -struct tape_builder; -struct tape_writer; - -} // namespace stage2 -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H -/* end file generic/stage2/base.h for rvv_vls */ -/* including generic/stage2/tape_writer.h for rvv_vls: #include */ -/* begin file generic/stage2/tape_writer.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace stage2 { - -struct tape_writer { - /** The next place to write to tape */ - uint64_t *next_tape_loc; - - /** Write a signed 64-bit value to tape. */ - simdjson_inline void append_s64(int64_t value) noexcept; - - /** Write an unsigned 64-bit value to tape. */ - simdjson_inline void append_u64(uint64_t value) noexcept; - - /** Write a double value to tape. */ - simdjson_inline void append_double(double value) noexcept; - - /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ - simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; - - /** - * Append a tape entry (an 8-bit type,and 56 bits worth of value). - */ - simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; - - /** - * Skip the current tape entry without writing. - * - * Used to skip the start of the container, since we'll come back later to fill it in when the - * container ends. - */ - simdjson_inline void skip() noexcept; - - /** - * Skip the number of tape entries necessary to write a large u64 or i64. - */ - simdjson_inline void skip_large_integer() noexcept; - - /** - * Skip the number of tape entries necessary to write a double. - */ - simdjson_inline void skip_double() noexcept; - - /** - * Write a value to a known location on tape. - * - * Used to go back and write out the start of a container after the container ends. - */ - simdjson_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; - -private: - /** - * Append both the tape entry, and a supplementary value following it. Used for types that need - * all 64 bits, such as double and uint64_t. - */ - template - simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; -}; // struct tape_writer - -simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { - append2(0, value, internal::tape_type::INT64); -} - -simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { - append(0, internal::tape_type::UINT64); - *next_tape_loc = value; - next_tape_loc++; -} - -/** Write a double value to tape. */ -simdjson_inline void tape_writer::append_double(double value) noexcept { - append2(0, value, internal::tape_type::DOUBLE); -} - -simdjson_inline void tape_writer::skip() noexcept { - next_tape_loc++; -} - -simdjson_inline void tape_writer::skip_large_integer() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::skip_double() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { - *next_tape_loc = val | ((uint64_t(char(t))) << 56); - next_tape_loc++; -} - -template -simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { - append(val, t); - static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); - memcpy(next_tape_loc, &val2, sizeof(val2)); - next_tape_loc++; -} - -simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { - tape_loc = val | ((uint64_t(char(t))) << 56); -} - -simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { - // Write to string buffer: [4-byte LE length][digits][null] - uint32_t str_len = uint32_t(len); - memcpy(string_buf, &str_len, sizeof(uint32_t)); - memcpy(string_buf + sizeof(uint32_t), src, len); - string_buf[sizeof(uint32_t) + len] = 0; - // Tape entry: offset into string buffer - // The caller must set the offset relative to doc.string_buf base - append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches - string_buf += sizeof(uint32_t) + len + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H -/* end file generic/stage2/tape_writer.h for rvv_vls */ -/* including generic/stage2/logger.h for rvv_vls: #include */ -/* begin file generic/stage2/logger.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - - -// This is for an internal-only stage 2 specific logger. -// Set LOG_ENABLED = true to log what stage 2 is doing! -namespace simdjson { -namespace rvv_vls { -namespace { -namespace logger { - - static constexpr const char * DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; - -#if SIMDJSON_VERBOSE_LOGGING - static constexpr const bool LOG_ENABLED = true; -#else - static constexpr const bool LOG_ENABLED = false; -#endif - static constexpr const int LOG_EVENT_LEN = 20; - static constexpr const int LOG_BUFFER_LEN = 30; - static constexpr const int LOG_SMALL_BUFFER_LEN = 10; - static constexpr const int LOG_INDEX_LEN = 5; - - static int log_depth; // Not threadsafe. Log only. - - // Helper to turn unprintable or newline characters into spaces - static simdjson_inline char printable_char(char c) { - if (c >= 0x20) { - return c; - } else { - return ' '; - } - } - - // Print the header and set up log_start - static simdjson_inline void log_start() { - if (LOG_ENABLED) { - log_depth = 0; - printf("\n"); - printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); - printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN+2, DASHES, LOG_BUFFER_LEN+2, DASHES, LOG_SMALL_BUFFER_LEN+2, DASHES, 5+2, DASHES); - } - } - - simdjson_unused static simdjson_inline void log_string(const char *message) { - if (LOG_ENABLED) { - printf("%s\n", message); - } - } - - // Logs a single line from the stage 2 DOM parser - template - static simdjson_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { - if (LOG_ENABLED) { - printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); - auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; - auto next_index = structurals.next_structural; - auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); - auto next = &structurals.buf[*next_index]; - { - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i=0;i */ -/* begin file generic/stage2/json_iterator.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace stage2 { - -class json_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - uint32_t depth{0}; - - /** - * Walk the JSON document. - * - * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as - * the first parameter; some callbacks have other parameters as well: - * - * - visit_document_start() - at the beginning. - * - visit_document_end() - at the end (if things were successful). - * - * - visit_array_start() - at the start `[` of a non-empty array. - * - visit_array_end() - at the end `]` of a non-empty array. - * - visit_empty_array() - when an empty array is encountered. - * - * - visit_object_end() - at the start `]` of a non-empty object. - * - visit_object_start() - at the end `]` of a non-empty object. - * - visit_empty_object() - when an empty object is encountered. - * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is - * guaranteed to point at the first quote of the string (`"key"`). - * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. - * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. - * - * - increment_count(iter) - each time a value is found in an array or object. - */ - template - simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; - - /** - * Create an iterator capable of walking a JSON document. - * - * The document must have already passed through stage 1. - */ - simdjson_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); - - /** - * Look at the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *peek(const uint32_t *batch_end = nullptr) const noexcept; - /** - * Advance to the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *advance(const uint32_t *batch_end = nullptr) noexcept; - /** - * Get the remaining length of the document, from the start of the current token. - */ - simdjson_inline size_t remaining_len() const noexcept; - /** - * Check if we are at the end of the document. - * - * If this is true, there are no more tokens. - */ - simdjson_inline bool at_eof() const noexcept; - /** - * Check if we are at the beginning of the document. - */ - simdjson_inline bool at_beginning() const noexcept; - simdjson_inline uint8_t last_structural() const noexcept; - - /** - * Log that a value has been found. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_value(const char *type) const noexcept; - /** - * Log the start of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_start_value(const char *type) const noexcept; - /** - * Log the end of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_end_value(const char *type) const noexcept; - /** - * Log an error. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_error(const char *error) const noexcept; - - template - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; - template - simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; -}; - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { - logger::log_start(); - - // - // Start the document - // - if (at_eof()) { return EMPTY; } - // Only used when BOUNDED (see advance()); a local, so that it can stay in a - // register, and vanishes from the other walks. - const uint32_t *const batch_end = &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - log_start_value("document"); - SIMDJSON_TRY( visitor.visit_document_start(*this) ); - - // - // Read first value - // - { - auto value = advance(batch_end); - - // Make sure the outer object or array is closed before continuing; otherwise, there are ways we - // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 - if (!STREAMING) { - switch (*value) { - case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; - case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; - } - } - - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; - } - } - goto document_end; - -// -// Object parser states -// -object_begin: - log_start_value("object"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = false; - SIMDJSON_TRY( visitor.visit_object_start(*this) ); - - { - auto key = advance(batch_end); - if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.increment_count(*this) ); - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - -object_field: - if (simdjson_unlikely( (*advance(batch_end) != ':') )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -object_continue: - switch (*advance(batch_end)) { - case ',': - SIMDJSON_TRY( visitor.increment_count(*this) ); - { - auto key = advance(batch_end); - if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - goto object_field; - case '}': log_end_value("object"); SIMDJSON_TRY( visitor.visit_object_end(*this) ); goto scope_end; - default: log_error("No comma between object fields"); return TAPE_ERROR; - } - -scope_end: - depth--; - if (depth == 0) { goto document_end; } - if (dom_parser.is_array[depth]) { goto array_continue; } - goto object_continue; - -// -// Array parser states -// -array_begin: - log_start_value("array"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = true; - SIMDJSON_TRY( visitor.visit_array_start(*this) ); - SIMDJSON_TRY( visitor.increment_count(*this) ); - -array_value: - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -array_continue: - switch (*advance(batch_end)) { - case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; - case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; - default: log_error("Missing comma between array values"); return TAPE_ERROR; - } - -document_end: - log_end_value("document"); - SIMDJSON_TRY( visitor.visit_document_end(*this) ); - - dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); - - // If we didn't make it to the end, it's an error - if ( !STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes ) { - log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); - return TAPE_ERROR; - } - - return SUCCESS; - -} // walk_document() - -simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { -} - -// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires -// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). -// -// When BOUNDED (see dom_parser_implementation::_bounded_batch), a document may -// not run past batch_end, one past the last structural index of the batch: -// past it, we return the same dummy, and walk_document fails cleanly. -template -simdjson_inline const uint8_t *json_iterator::peek(const uint32_t *batch_end) const noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -template -simdjson_inline const uint8_t *json_iterator::advance(const uint32_t *batch_end) noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural++); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -simdjson_inline size_t json_iterator::remaining_len() const noexcept { - return dom_parser.len - *(next_structural-1); -} - -simdjson_inline bool json_iterator::at_eof() const noexcept { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; -} -simdjson_inline bool json_iterator::at_beginning() const noexcept { - return next_structural == dom_parser.structural_indexes.get(); -} -simdjson_inline uint8_t json_iterator::last_structural() const noexcept { - return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; -} - -simdjson_inline void json_iterator::log_value(const char *type) const noexcept { - logger::log_line(*this, "", type, ""); -} - -simdjson_inline void json_iterator::log_start_value(const char *type) const noexcept { - logger::log_line(*this, "+", type, ""); - if (logger::LOG_ENABLED) { logger::log_depth++; } -} - -simdjson_inline void json_iterator::log_end_value(const char *type) const noexcept { - if (logger::LOG_ENABLED) { logger::log_depth--; } - logger::log_line(*this, "-", type, ""); -} - -simdjson_inline void json_iterator::log_error(const char *error) const noexcept { - logger::log_line(*this, "", "ERROR", error); -} - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { - switch (*value) { - case '"': return visitor.visit_root_string(*this, value); - case 't': return visitor.visit_root_true_atom(*this, value); - case 'f': return visitor.visit_root_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_root_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_root_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_root_inf_atom(*this, value); -#else - case 'n': return visitor.visit_root_null_atom(*this, value); -#endif - case '-': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return visitor.visit_root_number(*this, value); - default: - log_error("Document starts with a non-value character"); - return TAPE_ERROR; - } -} -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { - // Use the fact that most scalars are going to be either strings or numbers. - if(*value == '"') { - return visitor.visit_string(*this, value); - } else if (((*value - '0') < 10) || (*value == '-')) { - return visitor.visit_number(*this, value); - } - // true, false, null are uncommon. - switch (*value) { - case 't': return visitor.visit_true_atom(*this, value); - case 'f': return visitor.visit_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_inf_atom(*this, value); -#else - case 'n': return visitor.visit_null_atom(*this, value); -#endif - default: - log_error("Non-value found when value was expected!"); - return TAPE_ERROR; - } -} - -} // namespace stage2 -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H -/* end file generic/stage2/json_iterator.h for rvv_vls */ -/* including generic/stage2/stringparsing.h for rvv_vls: #include */ -/* begin file generic/stage2/stringparsing.h for rvv_vls */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace rvv_vls { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - // Decode adjacent Unicode escapes without returning to the - // quote-and-backslash scanner between code points. - do { - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } while (src[0] == '\\' && src[1] == 'u'); - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -/** - * Bounds-safe variant of parse_string for input buffers that are NOT padded to - * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input - * byte (buf + len). It behaves exactly like parse_string while we are at least - * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays - * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few - * remaining bytes into a space-padded scratch buffer and finishes with the - * regular parse_string. This keeps the delicate escape/Unicode handling in one - * place (the proven parse_string) rather than duplicating it. - * - * Correctness relies on stage 1 having validated the string, i.e. there is an - * unescaped closing quote within [src, buf_end); that quote is therefore inside - * the copied scratch, so parse_string finds it without running off the scratch. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { - // Far from the end: identical to parse_string's loop. The guard uses - // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past - // buf_end; escape/Unicode look-aheads read within the string (before the - // closing quote, which is < buf_end), so they are in bounds here too. - // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: - // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate - // following bytes (+6 from the '\'), then (if it sees \u) another - // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the - // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the - // chunk start. The +12 margin ensures that even on kernels where - // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays - // in-bounds. The scratch fallback (3*PAD) is already safe. - while (src + SIMDJSON_PADDING + 12 <= buf_end) { - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - if (bs_quote.has_quote_first()) { - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - if (escape_char == 'u') { - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - // Within the final SIMDJSON_PADDING bytes: copy what remains into a - // space-padded scratch (spaces are neither quote nor backslash, so they do not - // disturb matching) and let the regular parser finish from there. The closing - // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in - // the chunk starting at some offset <= remaining and reads at most - // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. - // We size at 3x for a comfortable margin (the unicode look-ahead reads a few - // extra bytes past an escape). - uint8_t scratch[SIMDJSON_PADDING * 3]; - const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING - std::memset(scratch, ' ', sizeof(scratch)); - std::memcpy(scratch, src, remaining); - return parse_string(scratch, dst, allow_replacement); -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for rvv_vls */ -/* including generic/stage2/tape_builder.h for rvv_vls: #include */ -/* begin file generic/stage2/tape_builder.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace stage2 { - -template -struct tape_builder_impl { - /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; - /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator &iter) noexcept; - - /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator &iter) noexcept; - /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator &iter) noexcept; - /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator &iter) noexcept; - - /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator &iter) noexcept; - /** - * Called when a key in a field is encountered. - * - * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array - * will be called after this with the field value. - */ - simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; - /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator &iter) noexcept; - /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator &iter) noexcept; - - /** - * Called when a string, number, boolean or null is found. - */ - simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; - /** - * Called when a string, number, boolean or null is found at the top level of a document (i.e. - * when there is no array or object and the entire document is a single string, number, boolean or - * null. - * - * This is separate from primitive() because simdjson's normal primitive parsing routines assume - * there is at least one more token after the value, which is only true in an array or object. - */ - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - -#if SIMDJSON_ENABLE_NAN_INF - simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, - // this returns a tape error on failure. - simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; -#endif - - /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; - - /** Next location to write to tape */ - tape_writer tape; -public: - simdjson_inline tape_builder_impl(dom::document &doc) noexcept; -private: - /** Next write location in the string buf for stage 2 parsing */ - uint8_t *current_string_buf_loc; - - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - simdjson_inline void start_container(json_iterator &iter) noexcept; - simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder_impl - -// Thin, non-templated entry so each architecture's stage2() keeps calling -// tape_builder::parse_document unchanged. It chooses the bounds-safe -// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the -// choice is a compile-time constant inside the walk: the padded path carries no -// extra branch or load (see tape_builder_impl::visit_string). -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } - - // The stage 2 entry points of dom::document_stream, which picks one per batch - // (see internal::dom_parser_implementation::stage2_next_function). Out of - // line, so that stage2_next() can share the code. - simdjson_warn_unused static simdjson_never_inline error_code parse_next_document( - internal::dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - return parse_document(static_cast(dom_parser), doc); - } - - // A document of a stream batch that stage 2 must bound to its structural - // indexes (see internal::dom_parser_implementation::_bounded_batch). - simdjson_warn_unused static simdjson_never_inline error_code parse_next_bounded_document( - internal::dom_parser_implementation &_dom_parser, dom::document &doc) noexcept { - dom_parser_implementation &dom_parser = static_cast(_dom_parser); - dom_parser.doc = &doc; - json_iterator iter(dom_parser, dom_parser.next_structural_index); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } -}; - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_root_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { - constexpr uint32_t start_tape_index = 0; - tape.append(start_tape_index, internal::tape_type::ROOT); - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { - return visit_string(iter, key, true); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 - return SUCCESS; -} - -template -simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { - iter.log_value(key ? "key" : "string"); - uint8_t *dst = on_start_string(iter); - // We do not allow replacement when the escape characters are invalid. - // UNPADDED is a compile-time constant chosen once per document by - // tape_builder::parse_document, so the padded build instantiates only the - // plain parse_string call below -- no runtime branch and no flag load. - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); - } else { - dst = stringparsing::parse_string(value+1, dst, false); - } - if (dst == nullptr) { - iter.log_error("Invalid escape in string"); - return STRING_ERROR; - } - on_end_string(dst); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { - return visit_string(iter, value); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("number"); - const uint8_t *num = value; - std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - // numberparsing reads ahead in 8-byte blocks for floats - // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a - // number whose digits reach the final bytes of an unpadded buffer would read - // past it. *(next_structural) is the offset of the token following this - // number, hence an upper bound on where the digits end; when that is within - // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail - // (mirroring visit_root_number). This fires only for numbers near the end. - if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { - const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document - copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, rl); - std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); - num = copy.get(); - } - } - error_code err = numberparsing::parse_number(num, tape); - if (simdjson_unlikely(err == BIGINT_ERROR && - iter.dom_parser._number_as_string)) { - // Write big integer to string buffer using the same format as strings. - // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = num; - if (*p == '-') p++; - while (numberparsing::is_digit(*p)) p++; - // The digit run must be terminated by a structural or whitespace character; otherwise the - // token is malformed (e.g. "123456789123456789123x"). - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - num); - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); - uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, num, len); - dst += len; - on_end_string(dst); - return SUCCESS; - } - return err; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { - // - // We need to make a copy to make sure that the string is space terminated. - // This is not about padding the input, which should already padded up - // to len + SIMDJSON_PADDING. However, we have no control at this stage - // on how the padding was done. What if the input string was padded with nulls? - // It is quite common for an input string to have an extra null character (C string). - // We do not want to allow 9\0 (where \0 is the null character) inside a JSON - // document, but the string "9\0" by itself is fine. So we make a copy and - // pad the input with spaces when we know that there is just one input element. - // This copy is relatively expensive, but it will almost never be called in - // practice unless you are in the strange scenario where you have many JSON - // documents made of single atoms. - // - // In a stream, the input goes on with other documents: copy up to the next - // structural only, not to the end of the batch. - const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); - std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, len); - std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); - error_code error = visit_number(iter, copy.get()); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated - // token at the very end of an unpadded buffer would over-read. Use the - // length-aware form there (the root variant already does this). - const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) - : atomparsing::is_valid_true_atom(value); - if (!ok) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) - : atomparsing::is_valid_false_atom(value); - if (!ok) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) - : atomparsing::is_valid_null_atom(value); - if (!ok) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -#if SIMDJSON_ENABLE_NAN_INF -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - // For unpadded input use the length-aware validator so the 'infinity'-style - // 8-byte compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) - : atomparsing::is_valid_nan_atom(value); - if (!ok) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. - // For unpadded input use the length-aware validator so the 'infinity' 8-byte - // compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) - : atomparsing::is_valid_inf_atom(value); - if (!ok) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure - if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -// private: - -template -simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { - return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - auto start_index = next_tape_index(iter); - tape.append(start_index+2, start); - tape.append(start_index, end); - return SUCCESS; -} - -template -simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); - iter.dom_parser.open_containers[iter.depth].count = 0; - tape.skip(); // We don't actually *write* the start element until the end. -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - // Write the ending tape element, pointing at the start location - const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; - tape.append(start_tape_index, end); - // Write the start tape element, pointing at the end location (and including count) - // count can overflow if it exceeds 24 bits... so we saturate - // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). - const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; - const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); - return SUCCESS; -} - -template -simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { - // we advance the point, accounting for the fact that we have a NULL termination - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); - return current_string_buf_loc + sizeof(uint32_t); -} - -template -simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { - uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); - // TODO check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); - // NULL termination is still handy if you expect all your strings to - // be NULL terminated? It comes at a small cost - *dst = 0; - current_string_buf_loc = dst + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H -/* end file generic/stage2/tape_builder.h for rvv_vls */ -/* end file generic/stage2/amalgamated.h for rvv_vls */ - -// -// Stage 1 -// -namespace simdjson { -namespace rvv_vls { - -simdjson_warn_unused error_code implementation::create_dom_parser_implementation( - size_t capacity, - size_t max_depth, - std::unique_ptr& dst -) const noexcept { - dst.reset( new (std::nothrow) dom_parser_implementation() ); - if (!dst) { return MEMALLOC; } - if (auto err = dst->set_capacity(capacity)) - return err; - if (auto err = dst->set_max_depth(max_depth)) - return err; - return SUCCESS; -} - -namespace { - -using namespace simd; - -simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - static const uint8_t wsTable[16] = { ' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100 }; - static const uint8_t opTable[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ':', '{', ',', '}', 0, 0 }; - vuint8_t vws = __riscv_vle8_v_u8m1(wsTable, 16); - vuint8_t vop = __riscv_vle8_v_u8m1(opTable, 16); - vuint8x64_t lo = __riscv_vand(in, 15, 64); - vuint8x64_t curl = __riscv_vor(in, 0x20, 64); - -#if __riscv_v_fixed_vlen == 128 - vboolx64_t mws = __riscv_vmseq(simdutf_vrgather_u8m1x4(vws, lo), in, 64); - vboolx64_t mop = __riscv_vmseq(simdutf_vrgather_u8m1x4(vop, lo), curl, 64); -#elif __riscv_v_fixed_vlen == 256 - vboolx64_t mws = __riscv_vmseq(simdutf_vrgather_u8m1x2(vws, lo), in, 64); - vboolx64_t mop = __riscv_vmseq(simdutf_vrgather_u8m1x2(vop, lo), curl, 64); -#else - vboolx64_t mws = __riscv_vmseq(__riscv_vrgather(vws, lo, 64), in, 64); - vboolx64_t mop = __riscv_vmseq(__riscv_vrgather(vop, lo, 64), curl, 64); -#endif - - return { - __riscv_vmv_x(__riscv_vreinterpret_u64m1(mws)), - __riscv_vmv_x(__riscv_vreinterpret_u64m1(mop)) - }; -} - -simdjson_inline bool is_ascii(const simd8x64& input) { - return input.is_ascii(); -} - -simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be >= 0x80 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be >= 0x80 - return is_third_byte | is_fourth_byte; -} - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -// -// Stage 2 -// - -// -// Implementation-specific overrides -// -namespace simdjson { -namespace rvv_vls { - -simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { - return rvv_vls::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, stage1_mode streaming) noexcept { - this->buf = _buf; - this->len = _len; - return rvv_vls::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); -} - -simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - return rvv_vls::stage1::generic_validate_utf8(buf,len); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_next_document(*this, _doc); -} - -internal::dom_parser_implementation::stage2_next_function_t dom_parser_implementation::stage2_next_function(bool bounded) const noexcept { - return bounded ? stage2::tape_builder::parse_next_bounded_document : stage2::tape_builder::parse_next_document; -} - -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept { - return rvv_vls::stringparsing::parse_string(src, dst, allow_replacement); -} - -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { - return rvv_vls::stringparsing::parse_wobbly_string(src, dst); -} - -simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { - auto error = stage1(_buf, _len, stage1_mode::regular); - if (error) { return error; } - return stage2(_doc); -} - -} // namespace rvv_vls -} // namespace simdjson - -/* including simdjson/rvv-vls/end.h: #include */ -/* begin file simdjson/rvv-vls/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* undefining SIMDJSON_IMPLEMENTATION from "rvv_vls" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/rvv-vls/end.h */ - -#endif // SIMDJSON_SRC_RVV_VLS_CPP -/* end file rvv-vls.cpp */ -#endif -#if SIMDJSON_IMPLEMENTATION_FALLBACK -/* including fallback.cpp: #include */ -/* begin file fallback.cpp */ -#ifndef SIMDJSON_SRC_FALLBACK_CPP -#define SIMDJSON_SRC_FALLBACK_CPP - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* including simdjson/fallback.h: #include */ -/* begin file simdjson/fallback.h */ -#ifndef SIMDJSON_FALLBACK_H -#define SIMDJSON_FALLBACK_H - -/* including simdjson/fallback/begin.h: #include "simdjson/fallback/begin.h" */ -/* begin file simdjson/fallback/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "fallback" */ -#define SIMDJSON_IMPLEMENTATION fallback -/* including simdjson/fallback/base.h: #include "simdjson/fallback/base.h" */ -/* begin file simdjson/fallback/base.h */ -#ifndef SIMDJSON_FALLBACK_BASE_H -#define SIMDJSON_FALLBACK_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * Fallback implementation (runs on any machine). - */ -namespace fallback { - -class implementation; - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_FALLBACK_BASE_H -/* end file simdjson/fallback/base.h */ -/* including simdjson/fallback/bitmanipulation.h: #include "simdjson/fallback/bitmanipulation.h" */ -/* begin file simdjson/fallback/bitmanipulation.h */ -#ifndef SIMDJSON_FALLBACK_BITMANIPULATION_H -#define SIMDJSON_FALLBACK_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { -namespace { - -#if defined(_MSC_VER) && !defined(_M_ARM64) && !defined(_M_X64) -static inline unsigned char _BitScanForward64(unsigned long* ret, uint64_t x) { - unsigned long x0 = (unsigned long)x, top, bottom; - _BitScanForward(&top, (unsigned long)(x >> 32)); - _BitScanForward(&bottom, x0); - *ret = x0 ? bottom : 32 + top; - return x != 0; -} -static unsigned char _BitScanReverse64(unsigned long* ret, uint64_t x) { - unsigned long x1 = (unsigned long)(x >> 32), top, bottom; - _BitScanReverse(&top, x1); - _BitScanReverse(&bottom, (unsigned long)x); - *ret = x1 ? top + 32 : bottom; - return x != 0; -} -#endif - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { -#ifdef _MSC_VER - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// _MSC_VER -} - -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - // Search the mask data from least significant bit (LSB) - // to most significant bit (MSB) for a set bit (1). - if (_BitScanForward64(&trailing_zero, input_num)) - return (int)trailing_zero; - else return 64; -#else - return __builtin_ctzll(input_num); -#endif// _MSC_VER -} - -} // unnamed namespace -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_FALLBACK_BITMANIPULATION_H -/* end file simdjson/fallback/bitmanipulation.h */ -/* including simdjson/fallback/stringparsing_defs.h: #include "simdjson/fallback/stringparsing_defs.h" */ -/* begin file simdjson/fallback/stringparsing_defs.h */ -#ifndef SIMDJSON_FALLBACK_STRINGPARSING_DEFS_H -#define SIMDJSON_FALLBACK_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { -namespace { - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 1; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return c == '"'; } - simdjson_inline bool has_backslash() { return c == '\\'; } - simdjson_inline int quote_index() { return c == '"' ? 0 : 1; } - simdjson_inline int backslash_index() { return c == '\\' ? 0 : 1; } - - uint8_t c; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // store to dest unconditionally - we can overwrite the bits we don't like later - dst[0] = src[0]; - return { src[0] }; -} - - -} // unnamed namespace -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_FALLBACK_STRINGPARSING_DEFS_H -/* end file simdjson/fallback/stringparsing_defs.h */ -/* including simdjson/fallback/numberparsing_defs.h: #include "simdjson/fallback/numberparsing_defs.h" */ -/* begin file simdjson/fallback/numberparsing_defs.h */ -#ifndef SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H -#define SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#ifdef JSON_TEST_NUMBERS // for unit testing -void found_invalid_number(const uint8_t *buf); -void found_integer(int64_t result, const uint8_t *buf); -void found_unsigned_integer(uint64_t result, const uint8_t *buf); -void found_float(double result, const uint8_t *buf); -#endif - -namespace simdjson { -namespace fallback { -namespace numberparsing { - -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const char *chars) { - uint64_t val; - memcpy(&val, chars, sizeof(uint64_t)); - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - return parse_eight_digits_unrolled(reinterpret_cast(chars)); -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace fallback -} // namespace simdjson - -#ifndef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_IS_BIG_ENDIAN -#define SIMDJSON_SWAR_NUMBER_PARSING 0 -#else -#define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif -#endif - -#endif // SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H -/* end file simdjson/fallback/numberparsing_defs.h */ -/* end file simdjson/fallback/begin.h */ -/* including simdjson/generic/amalgamated.h for fallback: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for fallback */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for fallback: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for fallback */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for fallback */ -/* including simdjson/generic/jsoncharutils.h for fallback: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for fallback */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for fallback */ -/* including simdjson/generic/atomparsing.h for fallback: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for fallback */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace fallback { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for fallback */ -/* including simdjson/generic/dom_parser_implementation.h for fallback: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for fallback */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace fallback -} // namespace simdjson - -namespace simdjson { -namespace fallback { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for fallback */ -/* including simdjson/generic/implementation_simdjson_result_base.h for fallback: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for fallback */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for fallback */ -/* including simdjson/generic/numberparsing.h for fallback: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for fallback */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace fallback { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for fallback */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for fallback: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for fallback */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for fallback */ -/* end file simdjson/generic/amalgamated.h for fallback */ -/* including simdjson/fallback/end.h: #include "simdjson/fallback/end.h" */ -/* begin file simdjson/fallback/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* undefining SIMDJSON_IMPLEMENTATION from "fallback" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/fallback/end.h */ - -#endif // SIMDJSON_FALLBACK_H -/* end file simdjson/fallback.h */ -/* including simdjson/fallback/implementation.h: #include */ -/* begin file simdjson/fallback/implementation.h */ -#ifndef SIMDJSON_FALLBACK_IMPLEMENTATION_H -#define SIMDJSON_FALLBACK_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation( - "fallback", - "Generic fallback implementation", - 0 - ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_FALLBACK_IMPLEMENTATION_H -/* end file simdjson/fallback/implementation.h */ - -/* including simdjson/fallback/begin.h: #include */ -/* begin file simdjson/fallback/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "fallback" */ -#define SIMDJSON_IMPLEMENTATION fallback -/* including simdjson/fallback/base.h: #include "simdjson/fallback/base.h" */ -/* begin file simdjson/fallback/base.h */ -#ifndef SIMDJSON_FALLBACK_BASE_H -#define SIMDJSON_FALLBACK_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * Fallback implementation (runs on any machine). - */ -namespace fallback { - -class implementation; - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_FALLBACK_BASE_H -/* end file simdjson/fallback/base.h */ -/* including simdjson/fallback/bitmanipulation.h: #include "simdjson/fallback/bitmanipulation.h" */ -/* begin file simdjson/fallback/bitmanipulation.h */ -#ifndef SIMDJSON_FALLBACK_BITMANIPULATION_H -#define SIMDJSON_FALLBACK_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { -namespace { - -#if defined(_MSC_VER) && !defined(_M_ARM64) && !defined(_M_X64) -static inline unsigned char _BitScanForward64(unsigned long* ret, uint64_t x) { - unsigned long x0 = (unsigned long)x, top, bottom; - _BitScanForward(&top, (unsigned long)(x >> 32)); - _BitScanForward(&bottom, x0); - *ret = x0 ? bottom : 32 + top; - return x != 0; -} -static unsigned char _BitScanReverse64(unsigned long* ret, uint64_t x) { - unsigned long x1 = (unsigned long)(x >> 32), top, bottom; - _BitScanReverse(&top, x1); - _BitScanReverse(&bottom, (unsigned long)x); - *ret = x1 ? top + 32 : bottom; - return x != 0; -} -#endif - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { -#ifdef _MSC_VER - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// _MSC_VER -} - -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - // Search the mask data from least significant bit (LSB) - // to most significant bit (MSB) for a set bit (1). - if (_BitScanForward64(&trailing_zero, input_num)) - return (int)trailing_zero; - else return 64; -#else - return __builtin_ctzll(input_num); -#endif// _MSC_VER -} - -} // unnamed namespace -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_FALLBACK_BITMANIPULATION_H -/* end file simdjson/fallback/bitmanipulation.h */ -/* including simdjson/fallback/stringparsing_defs.h: #include "simdjson/fallback/stringparsing_defs.h" */ -/* begin file simdjson/fallback/stringparsing_defs.h */ -#ifndef SIMDJSON_FALLBACK_STRINGPARSING_DEFS_H -#define SIMDJSON_FALLBACK_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { -namespace { - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 1; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return c == '"'; } - simdjson_inline bool has_backslash() { return c == '\\'; } - simdjson_inline int quote_index() { return c == '"' ? 0 : 1; } - simdjson_inline int backslash_index() { return c == '\\' ? 0 : 1; } - - uint8_t c; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // store to dest unconditionally - we can overwrite the bits we don't like later - dst[0] = src[0]; - return { src[0] }; -} - - -} // unnamed namespace -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_FALLBACK_STRINGPARSING_DEFS_H -/* end file simdjson/fallback/stringparsing_defs.h */ -/* including simdjson/fallback/numberparsing_defs.h: #include "simdjson/fallback/numberparsing_defs.h" */ -/* begin file simdjson/fallback/numberparsing_defs.h */ -#ifndef SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H -#define SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#ifdef JSON_TEST_NUMBERS // for unit testing -void found_invalid_number(const uint8_t *buf); -void found_integer(int64_t result, const uint8_t *buf); -void found_unsigned_integer(uint64_t result, const uint8_t *buf); -void found_float(double result, const uint8_t *buf); -#endif - -namespace simdjson { -namespace fallback { -namespace numberparsing { - -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const char *chars) { - uint64_t val; - memcpy(&val, chars, sizeof(uint64_t)); - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - return parse_eight_digits_unrolled(reinterpret_cast(chars)); -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace fallback -} // namespace simdjson - -#ifndef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_IS_BIG_ENDIAN -#define SIMDJSON_SWAR_NUMBER_PARSING 0 -#else -#define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif -#endif - -#endif // SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H -/* end file simdjson/fallback/numberparsing_defs.h */ -/* end file simdjson/fallback/begin.h */ -/* including generic/stage1/find_next_document_index.h for fallback: #include */ -/* begin file generic/stage1/find_next_document_index.h for fallback */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { -namespace { -namespace stage1 { - -/** - * This algorithm is used to quickly identify the last structural position that - * makes up a complete document. - * - * It does this by going backwards and finding the last *document boundary* (a - * place where one value follows another without a comma between them). If the - * last document (the characters after the boundary) has an equal number of - * start and end brackets, it is considered complete. - * - * Simply put, we iterate over the structural characters, starting from - * the end. We consider that we found the end of a JSON document when the - * first element of the pair is NOT one of these characters: '{' '[' ':' ',' - * and when the second element is NOT one of these characters: '}' ']' ':' ','. - * - * This simple comparison works most of the time, but it does not cover cases - * where the batch's structural indexes contain a perfect amount of documents. - * In such a case, we do not have access to the structural index which follows - * the last document, therefore, we do not have access to the second element in - * the pair, and that means we cannot identify the last document. To fix this - * issue, we keep a count of the open and closed curly/square braces we found - * while searching for the pair. When we find a pair AND the count of open and - * closed curly/square braces is the same, we know that we just passed a - * complete document, therefore the last json buffer location is the end of the - * batch. - */ -simdjson_inline bool ends_with_partial_scalar(dom_parser_implementation &parser, size_t len) { - const uint8_t f = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - const uint8_t e = parser.buf[len - 1]; - return f != '{' && f != '[' && f != '}' && f != ']' && f != ':' && f != ',' && f != '"' && - e != ' ' && e != '\t' && e != '\n' && e != '\r'; -} - -/** - * The structural index past the last one of a partial batch is the end of this - * window, not a token of it: the byte there is the first byte of the next - * window. A finished value must not be rejected because that byte is ',', ':', - * ']' or '}'. A last document that itself ends with '{', '[', ',' or ':' is - * invalid JSON, yet its brackets may balance: walking it, the On-Demand API - * could read the next batch as a value. We leave such a document to the next - * batch, so that the batch ends between two documents. - */ -simdjson_inline bool ends_with_unfinished_document(dom_parser_implementation &parser) { - const uint8_t last = parser.buf[parser.structural_indexes[parser.n_structural_indexes - 1]]; - return last == '{' || last == '[' || last == ',' || last == ':'; -} - -simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser, bool defer_last = false) { - // Variant: do not count separately, just figure out depth - if(parser.n_structural_indexes == 0) { return 0; } - auto arr_cnt = 0; - auto obj_cnt = 0; - for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { - auto idxb = parser.structural_indexes[i]; - switch (parser.buf[idxb]) { - case ':': - case ',': - continue; - case '}': - obj_cnt--; - continue; - case ']': - arr_cnt--; - continue; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - auto idxa = parser.structural_indexes[i - 1]; - switch (parser.buf[idxa]) { - case '{': - case '[': - case ':': - case ',': - continue; - } - // Last document is complete, so the next document will appear after! - if (!arr_cnt && !obj_cnt) { - return defer_last ? i : parser.n_structural_indexes; - } - // Last document is incomplete; mark the document at i + 1 as the next one - return i; - } - // If we made it to the end, we want to finish counting to see if we have a full document. - switch (parser.buf[parser.structural_indexes[0]]) { - case '}': - obj_cnt--; - break; - case ']': - arr_cnt--; - break; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - if (!arr_cnt && !obj_cnt) { - // We have a complete document. - return defer_last ? 0 : parser.n_structural_indexes; - } - return 0; -} - -/** - * Sentinel value returned to indicate a document started but didn't fit - * (CAPACITY error), as opposed to 0 which means no document content found - * (EMPTY). - */ -constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; - -/** - * For RFC 7464 JSON text sequences, filter RS from structural indexes and - * find batch boundaries. - * - * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. - * RS bytes appear in structural_indexes as they are classified as scalars. - * This function: - * 1. Scans structural_indexes to find and count RS positions - * 2. Filters RS out of structural_indexes in-place - * 3. Determines batch boundaries based on RS positions - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @param scan_len Offset past which no value start may be derived. Equals len - * unless stage 1 dropped a trailing unclosed string, whose bytes it - * never validated. - * @return The number of structural indexes to keep (after RS filtering), - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t find_next_document_index_json_sequence( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start, - size_t scan_len) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Phase 1: Scan structural_indexes to find RS positions and handle them. - // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS - // is already in structural_indexes (it's an operator). For scalars like numbers, - // the digit following RS is NOT in structural_indexes because the scanner sees - // RS as a scalar, making the digit a scalar continuation, not a start. - // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the - // actual value start position. - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_rs_pos = 0; - uint32_t rs_count = 0; - - for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { - const uint32_t pos = parser.structural_indexes[read_idx]; - if (parser.buf[pos] == 0x1E) { - // This is an RS character - find the actual JSON value start. - last_rs_pos = pos; - rs_count++; - // Skip past this RS and any whitespace *and any additional RSes* - // to locate the real value. Consecutive RSes are degenerate - // "empty records" per RFC 7464; we collapse them here. They do - // not always appear as separate entries in structural_indexes - // because the scanner groups runs of adjacent non-whitespace - // scalar bytes (including RS) into a single scalar start. - uint32_t value_start = pos + 1; - while (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { - value_start++; - } else if (c == 0x1E) { - // Collapsed empty record. Still count it so rs_count reflects - // the true number of record markers and last_rs_pos tracks - // the final one. - last_rs_pos = value_start; - rs_count++; - value_start++; - } else { - break; - } - } - // If the scanner emitted additional structurals inside the - // whitespace+RS run we just walked over (i.e., isolated RSes - // separated by whitespace), skip past them so we do not - // double-count or double-emit. - while (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] < value_start) { - read_idx++; - } - // Check if the value start is an operator (always present in - // scanner structural_indexes) or a scalar-like start (which may - // be missing from structural_indexes and must be added here). - // Note: '"' is NOT always in structural_indexes. The scanner - // classifies '"' as a scalar character and emits it as a - // structural only when it is a *scalar start* (preceded by - // whitespace or an operator). When '"' immediately follows an - // RS (which the scanner also classifies as scalar), it is - // treated as a scalar continuation and not emitted - so we - // must add it here just like any other scalar value. - if (value_start < scan_len) { - const uint8_t c = parser.buf[value_start]; - const bool is_operator = - (c == '{' || c == '}' || c == '[' || c == ']' || - c == ':' || c == ','); - // If the next scanner structural is exactly at value_start, - // the scanner already emitted it (it followed whitespace) and - // we must not add a duplicate - a subsequent iteration will - // copy it into write_idx. - const bool already_emitted = - (read_idx + 1 < parser.n_structural_indexes && - parser.structural_indexes[read_idx + 1] == value_start); - if (!is_operator && !already_emitted) { - // Scalar value (number/true/false/null/string) - add its - // position since scanner missed it. - parser.structural_indexes[write_idx++] = value_start; - } - } - } else { - // Not RS, copy to output - parser.structural_indexes[write_idx++] = pos; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { - // Only RS markers here: the last one opens a record continuing past the - // window, so that is where the next batch resumes. - if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } - return 0; - } - if (rs_count == 0) { - // No RS found; for final batch, try generic boundary detection - return is_final ? find_next_document_index(parser) : 0; - } - - // Phase 2: Determine batch boundaries based on RS positions - - if (is_final) { - // Final batch: all documents are complete (last one ends at EOF). - // In json_sequence mode, RS markers define document boundaries, so all - // remaining structurals form complete documents. Return them all directly. - // (Calling find_next_document_index() would fail for scalar documents.) - return parser.n_structural_indexes; - } - - // Partial batch: need to find complete documents only. - // A document starting at an RS is complete if there is another RS after it. - next_batch_start = last_rs_pos; - - if (rs_count < 2) { - // Only one RS, so we have at most one document that may be incomplete. - // We cannot confirm it is complete without another RS. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least 2 RS markers. The last complete document ends before last_rs_pos. - - // Find the structural index cutoff: keep only structurals < last_rs_pos. - // Since we already filtered RS, all remaining structurals are valid. - // We iterate backward to find the last structural before last_rs_pos. - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_rs_pos) { - keep_count = i; - break; - } - } - - // No structurals before the last RS - no complete documents - if (keep_count == 0) { return 0; } - - // All documents before the last RS are complete by definition (the next RS - // confirms their end). No need to call find_next_document_index() which - // would fail for scalar documents like `1` or `"hello"`. - return keep_count; -} - -/** - * Filter comma-delimited documents by removing root-level commas from - * structural indexes. - * - * For comma-delimited format like `{...},{...},{...}`, we need to remove - * the commas that separate documents (depth 0) while preserving commas - * inside arrays and objects (depth > 0). - * - * After filtering, the structural indexes look like whitespace-delimited - * documents, so find_next_document_index() works unchanged. - * - * @param parser The parser with structural_indexes and buf. - * @param len The length of the current batch buffer. - * @param is_final True if this is the final batch (no more data coming). - * @param next_batch_start Output: offset where the next batch should start. - * @return The number of structural indexes to keep, - * 0 if no document content found (EMPTY), - * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). - */ -simdjson_inline uint32_t filter_comma_delimited( - dom_parser_implementation &parser, - size_t len, - bool is_final, - uint32_t &next_batch_start) { - // Default: next batch starts at end of buffer - next_batch_start = uint32_t(len); - - if (parser.n_structural_indexes == 0) { return 0; } - - // Track depth to identify root-level commas (depth 0) - int depth = 0; - // The EOF sentinel: len, or where a discarded unclosed string starts. - const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; - uint32_t write_idx = 0; - uint32_t last_root_comma_pos = 0; - uint32_t root_comma_count = 0; - - for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { - uint32_t idx = parser.structural_indexes[i]; - uint8_t c = parser.buf[idx]; - - switch (c) { - case '{': case '[': - depth++; - parser.structural_indexes[write_idx++] = idx; - break; - case '}': case ']': - depth--; - parser.structural_indexes[write_idx++] = idx; - break; - case ',': - if (depth == 0) { - // Root-level comma = document boundary, skip it - last_root_comma_pos = idx; - root_comma_count++; - continue; - } - parser.structural_indexes[write_idx++] = idx; - break; - default: - // Colons, scalars, etc. - parser.structural_indexes[write_idx++] = idx; - break; - } - } - - // Update structural index count. Compaction left a stale index in the slot - // past the end: restore the EOF sentinel that stage 1 had planted there, which - // document_stream::truncated_bytes() reads after a final batch. - parser.n_structural_indexes = write_idx; - parser.structural_indexes[write_idx] = sentinel; - - if (parser.n_structural_indexes == 0) { return 0; } - - if (is_final) { - // Final batch: use standard boundary detection on filtered indexes - return find_next_document_index(parser); - } - - // Partial batch: need to find complete documents only. - // A document ending with a root comma is complete. - if (root_comma_count == 0) { - // No root commas found; we cannot confirm any document is complete. - // The whole batch might be one incomplete document. - // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. - return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; - } - - // We have at least one root comma. Documents before the last comma are complete. - next_batch_start = last_root_comma_pos + 1; - - // Find the structural index cutoff: keep only structurals < last_root_comma_pos - uint32_t keep_count = 0; - for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { - if (parser.structural_indexes[i - 1] < last_root_comma_pos) { - keep_count = i; - break; - } - } - - if (keep_count == 0) { return 0; } - - // Use standard boundary detection on the complete portion - parser.n_structural_indexes = keep_count; - return find_next_document_index(parser); -} - -} // namespace stage1 -} // unnamed namespace -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H -/* end file generic/stage1/find_next_document_index.h for fallback */ -/* including generic/stage2/stringparsing.h for fallback: #include */ -/* begin file generic/stage2/stringparsing.h for fallback */ -#include -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace fallback { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if(!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -// handle a unicode codepoint using the wobbly convention -// https://simonsapin.github.io/wtf-8/ -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion is not valid; we defer the check for this to inside the - // multilingual plane check. - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t *src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - - -/** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - // Decode adjacent Unicode escapes without returning to the - // quote-and-backslash scanner between code points. - do { - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } while (src[0] == '\\' && src[1] == 'u'); - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -/** - * Bounds-safe variant of parse_string for input buffers that are NOT padded to - * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input - * byte (buf + len). It behaves exactly like parse_string while we are at least - * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays - * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few - * remaining bytes into a space-padded scratch buffer and finishes with the - * regular parse_string. This keeps the delicate escape/Unicode handling in one - * place (the proven parse_string) rather than duplicating it. - * - * Correctness relies on stage 1 having validated the string, i.e. there is an - * unescaped closing quote within [src, buf_end); that quote is therefore inside - * the copied scratch, so parse_string finds it without running off the scratch. - */ -simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { - // Far from the end: identical to parse_string's loop. The guard uses - // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past - // buf_end; escape/Unicode look-aheads read within the string (before the - // closing quote, which is < buf_end), so they are in bounds here too. - // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: - // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate - // following bytes (+6 from the '\'), then (if it sees \u) another - // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the - // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the - // chunk start. The +12 margin ensures that even on kernels where - // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays - // in-bounds. The scratch fallback (3*PAD) is already safe. - while (src + SIMDJSON_PADDING + 12 <= buf_end) { - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - if (bs_quote.has_quote_first()) { - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - if (escape_char == 'u') { - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } else { - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - // Within the final SIMDJSON_PADDING bytes: copy what remains into a - // space-padded scratch (spaces are neither quote nor backslash, so they do not - // disturb matching) and let the regular parser finish from there. The closing - // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in - // the chunk starting at some offset <= remaining and reads at most - // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. - // We size at 3x for a comfortable margin (the unicode look-ahead reads a few - // extra bytes past an escape). - uint8_t scratch[SIMDJSON_PADDING * 3]; - const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING - std::memset(scratch, ' ', sizeof(scratch)); - std::memcpy(scratch, src, remaining); - return parse_string(scratch, dst, allow_replacement); -} - -simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto b = backslash_and_quote{}; - auto bs_quote = b.copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } -} - -} // namespace stringparsing - -} // unnamed namespace -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for fallback */ -/* including generic/stage2/logger.h for fallback: #include */ -/* begin file generic/stage2/logger.h for fallback */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - - -// This is for an internal-only stage 2 specific logger. -// Set LOG_ENABLED = true to log what stage 2 is doing! -namespace simdjson { -namespace fallback { -namespace { -namespace logger { - - static constexpr const char * DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; - -#if SIMDJSON_VERBOSE_LOGGING - static constexpr const bool LOG_ENABLED = true; -#else - static constexpr const bool LOG_ENABLED = false; -#endif - static constexpr const int LOG_EVENT_LEN = 20; - static constexpr const int LOG_BUFFER_LEN = 30; - static constexpr const int LOG_SMALL_BUFFER_LEN = 10; - static constexpr const int LOG_INDEX_LEN = 5; - - static int log_depth; // Not threadsafe. Log only. - - // Helper to turn unprintable or newline characters into spaces - static simdjson_inline char printable_char(char c) { - if (c >= 0x20) { - return c; - } else { - return ' '; - } - } - - // Print the header and set up log_start - static simdjson_inline void log_start() { - if (LOG_ENABLED) { - log_depth = 0; - printf("\n"); - printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); - printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN+2, DASHES, LOG_BUFFER_LEN+2, DASHES, LOG_SMALL_BUFFER_LEN+2, DASHES, 5+2, DASHES); - } - } - - simdjson_unused static simdjson_inline void log_string(const char *message) { - if (LOG_ENABLED) { - printf("%s\n", message); - } - } - - // Logs a single line from the stage 2 DOM parser - template - static simdjson_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { - if (LOG_ENABLED) { - printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); - auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; - auto next_index = structurals.next_structural; - auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); - auto next = &structurals.buf[*next_index]; - { - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i=0;i */ -/* begin file generic/stage2/json_iterator.h for fallback */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { -namespace { -namespace stage2 { - -class json_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - uint32_t depth{0}; - - /** - * Walk the JSON document. - * - * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as - * the first parameter; some callbacks have other parameters as well: - * - * - visit_document_start() - at the beginning. - * - visit_document_end() - at the end (if things were successful). - * - * - visit_array_start() - at the start `[` of a non-empty array. - * - visit_array_end() - at the end `]` of a non-empty array. - * - visit_empty_array() - when an empty array is encountered. - * - * - visit_object_end() - at the start `]` of a non-empty object. - * - visit_object_start() - at the end `]` of a non-empty object. - * - visit_empty_object() - when an empty object is encountered. - * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is - * guaranteed to point at the first quote of the string (`"key"`). - * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. - * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. - * - * - increment_count(iter) - each time a value is found in an array or object. - */ - template - simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; - - /** - * Create an iterator capable of walking a JSON document. - * - * The document must have already passed through stage 1. - */ - simdjson_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); - - /** - * Look at the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *peek(const uint32_t *batch_end = nullptr) const noexcept; - /** - * Advance to the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - template - simdjson_inline const uint8_t *advance(const uint32_t *batch_end = nullptr) noexcept; - /** - * Get the remaining length of the document, from the start of the current token. - */ - simdjson_inline size_t remaining_len() const noexcept; - /** - * Check if we are at the end of the document. - * - * If this is true, there are no more tokens. - */ - simdjson_inline bool at_eof() const noexcept; - /** - * Check if we are at the beginning of the document. - */ - simdjson_inline bool at_beginning() const noexcept; - simdjson_inline uint8_t last_structural() const noexcept; - - /** - * Log that a value has been found. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_value(const char *type) const noexcept; - /** - * Log the start of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_start_value(const char *type) const noexcept; - /** - * Log the end of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_end_value(const char *type) const noexcept; - /** - * Log an error. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_error(const char *error) const noexcept; - - template - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; - template - simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; -}; - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { - logger::log_start(); - - // - // Start the document - // - if (at_eof()) { return EMPTY; } - // Only used when BOUNDED (see advance()); a local, so that it can stay in a - // register, and vanishes from the other walks. - const uint32_t *const batch_end = &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - log_start_value("document"); - SIMDJSON_TRY( visitor.visit_document_start(*this) ); - - // - // Read first value - // - { - auto value = advance(batch_end); - - // Make sure the outer object or array is closed before continuing; otherwise, there are ways we - // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 - if (!STREAMING) { - switch (*value) { - case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; - case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; - } - } - - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; - } - } - goto document_end; - -// -// Object parser states -// -object_begin: - log_start_value("object"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = false; - SIMDJSON_TRY( visitor.visit_object_start(*this) ); - - { - auto key = advance(batch_end); - if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.increment_count(*this) ); - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - -object_field: - if (simdjson_unlikely( (*advance(batch_end) != ':') )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -object_continue: - switch (*advance(batch_end)) { - case ',': - SIMDJSON_TRY( visitor.increment_count(*this) ); - { - auto key = advance(batch_end); - if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } - SIMDJSON_TRY( visitor.visit_key(*this, key) ); - } - goto object_field; - case '}': log_end_value("object"); SIMDJSON_TRY( visitor.visit_object_end(*this) ); goto scope_end; - default: log_error("No comma between object fields"); return TAPE_ERROR; - } - -scope_end: - depth--; - if (depth == 0) { goto document_end; } - if (dom_parser.is_array[depth]) { goto array_continue; } - goto object_continue; - -// -// Array parser states -// -array_begin: - log_start_value("array"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = true; - SIMDJSON_TRY( visitor.visit_array_start(*this) ); - SIMDJSON_TRY( visitor.increment_count(*this) ); - -array_value: - { - auto value = advance(batch_end); - switch (*value) { - case '{': if (*peek(batch_end) == '}') { advance(batch_end); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek(batch_end) == ']') { advance(batch_end); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; - default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; - } - } - -array_continue: - switch (*advance(batch_end)) { - case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; - case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; - default: log_error("Missing comma between array values"); return TAPE_ERROR; - } - -document_end: - log_end_value("document"); - SIMDJSON_TRY( visitor.visit_document_end(*this) ); - - dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); - - // If we didn't make it to the end, it's an error - if ( !STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes ) { - log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); - return TAPE_ERROR; - } - - return SUCCESS; - -} // walk_document() - -simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { -} - -// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires -// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). -// -// When BOUNDED (see dom_parser_implementation::_bounded_batch), a document may -// not run past batch_end, one past the last structural index of the batch: -// past it, we return the same dummy, and walk_document fails cleanly. -template -simdjson_inline const uint8_t *json_iterator::peek(const uint32_t *batch_end) const noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -template -simdjson_inline const uint8_t *json_iterator::advance(const uint32_t *batch_end) noexcept { - SIMDJSON_IF_CONSTEXPR (BOUNDED) { - if (simdjson_unlikely(next_structural >= batch_end)) { - static constexpr uint8_t streaming_eof_sentinel = 0; - return &streaming_eof_sentinel; - } - } - const uint32_t idx = *(next_structural++); - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - if (simdjson_unlikely(idx >= dom_parser.len)) { - static constexpr uint8_t unpadded_eof_sentinel = 0; - return &unpadded_eof_sentinel; - } - } - return &buf[idx]; -} -simdjson_inline size_t json_iterator::remaining_len() const noexcept { - return dom_parser.len - *(next_structural-1); -} - -simdjson_inline bool json_iterator::at_eof() const noexcept { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; -} -simdjson_inline bool json_iterator::at_beginning() const noexcept { - return next_structural == dom_parser.structural_indexes.get(); -} -simdjson_inline uint8_t json_iterator::last_structural() const noexcept { - return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; -} - -simdjson_inline void json_iterator::log_value(const char *type) const noexcept { - logger::log_line(*this, "", type, ""); -} - -simdjson_inline void json_iterator::log_start_value(const char *type) const noexcept { - logger::log_line(*this, "+", type, ""); - if (logger::LOG_ENABLED) { logger::log_depth++; } -} - -simdjson_inline void json_iterator::log_end_value(const char *type) const noexcept { - if (logger::LOG_ENABLED) { logger::log_depth--; } - logger::log_line(*this, "-", type, ""); -} - -simdjson_inline void json_iterator::log_error(const char *error) const noexcept { - logger::log_line(*this, "", "ERROR", error); -} - -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { - switch (*value) { - case '"': return visitor.visit_root_string(*this, value); - case 't': return visitor.visit_root_true_atom(*this, value); - case 'f': return visitor.visit_root_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_root_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_root_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_root_inf_atom(*this, value); -#else - case 'n': return visitor.visit_root_null_atom(*this, value); -#endif - case '-': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return visitor.visit_root_number(*this, value); - default: - log_error("Document starts with a non-value character"); - return TAPE_ERROR; - } -} -template -simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { - // Use the fact that most scalars are going to be either strings or numbers. - if(*value == '"') { - return visitor.visit_string(*this, value); - } else if (((*value - '0') < 10) || (*value == '-')) { - return visitor.visit_number(*this, value); - } - // true, false, null are uncommon. - switch (*value) { - case 't': return visitor.visit_true_atom(*this, value); - case 'f': return visitor.visit_false_atom(*this, value); -#if SIMDJSON_ENABLE_NAN_INF - case 'n': { - auto err = visitor.visit_null_atom(*this, value); - if (err == SUCCESS) { return err; } - // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails - return visitor.visit_nan_atom(*this, value, err); - } - // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs - case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); - case 'i': - case 'I': return visitor.visit_inf_atom(*this, value); -#else - case 'n': return visitor.visit_null_atom(*this, value); -#endif - default: - log_error("Non-value found when value was expected!"); - return TAPE_ERROR; - } -} - -} // namespace stage2 -} // unnamed namespace -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H -/* end file generic/stage2/json_iterator.h for fallback */ -/* including generic/stage2/tape_writer.h for fallback: #include */ -/* begin file generic/stage2/tape_writer.h for fallback */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace fallback { -namespace { -namespace stage2 { - -struct tape_writer { - /** The next place to write to tape */ - uint64_t *next_tape_loc; - - /** Write a signed 64-bit value to tape. */ - simdjson_inline void append_s64(int64_t value) noexcept; - - /** Write an unsigned 64-bit value to tape. */ - simdjson_inline void append_u64(uint64_t value) noexcept; - - /** Write a double value to tape. */ - simdjson_inline void append_double(double value) noexcept; - - /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ - simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; - - /** - * Append a tape entry (an 8-bit type,and 56 bits worth of value). - */ - simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; - - /** - * Skip the current tape entry without writing. - * - * Used to skip the start of the container, since we'll come back later to fill it in when the - * container ends. - */ - simdjson_inline void skip() noexcept; - - /** - * Skip the number of tape entries necessary to write a large u64 or i64. - */ - simdjson_inline void skip_large_integer() noexcept; - - /** - * Skip the number of tape entries necessary to write a double. - */ - simdjson_inline void skip_double() noexcept; - - /** - * Write a value to a known location on tape. - * - * Used to go back and write out the start of a container after the container ends. - */ - simdjson_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; - -private: - /** - * Append both the tape entry, and a supplementary value following it. Used for types that need - * all 64 bits, such as double and uint64_t. - */ - template - simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; -}; // struct tape_writer - -simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { - append2(0, value, internal::tape_type::INT64); -} - -simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { - append(0, internal::tape_type::UINT64); - *next_tape_loc = value; - next_tape_loc++; -} - -/** Write a double value to tape. */ -simdjson_inline void tape_writer::append_double(double value) noexcept { - append2(0, value, internal::tape_type::DOUBLE); -} - -simdjson_inline void tape_writer::skip() noexcept { - next_tape_loc++; -} - -simdjson_inline void tape_writer::skip_large_integer() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::skip_double() noexcept { - next_tape_loc += 2; -} - -simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { - *next_tape_loc = val | ((uint64_t(char(t))) << 56); - next_tape_loc++; -} - -template -simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { - append(val, t); - static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); - memcpy(next_tape_loc, &val2, sizeof(val2)); - next_tape_loc++; -} - -simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { - tape_loc = val | ((uint64_t(char(t))) << 56); -} - -simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { - // Write to string buffer: [4-byte LE length][digits][null] - uint32_t str_len = uint32_t(len); - memcpy(string_buf, &str_len, sizeof(uint32_t)); - memcpy(string_buf + sizeof(uint32_t), src, len); - string_buf[sizeof(uint32_t) + len] = 0; - // Tape entry: offset into string buffer - // The caller must set the offset relative to doc.string_buf base - append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches - string_buf += sizeof(uint32_t) + len + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H -/* end file generic/stage2/tape_writer.h for fallback */ -/* including generic/stage2/tape_builder.h for fallback: #include */ -/* begin file generic/stage2/tape_builder.h for fallback */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - - -namespace simdjson { -namespace fallback { -namespace { -namespace stage2 { - -template -struct tape_builder_impl { - /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; - /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator &iter) noexcept; - - /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator &iter) noexcept; - /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator &iter) noexcept; - /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator &iter) noexcept; - - /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator &iter) noexcept; - /** - * Called when a key in a field is encountered. - * - * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array - * will be called after this with the field value. - */ - simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; - /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator &iter) noexcept; - /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator &iter) noexcept; - - /** - * Called when a string, number, boolean or null is found. - */ - simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; - /** - * Called when a string, number, boolean or null is found at the top level of a document (i.e. - * when there is no array or object and the entire document is a single string, number, boolean or - * null. - * - * This is separate from primitive() because simdjson's normal primitive parsing routines assume - * there is at least one more token after the value, which is only true in an array or object. - */ - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - -#if SIMDJSON_ENABLE_NAN_INF - simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; - // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, - // this returns a tape error on failure. - simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; -#endif - - /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; - - /** Next location to write to tape */ - tape_writer tape; -public: - simdjson_inline tape_builder_impl(dom::document &doc) noexcept; -private: - /** Next write location in the string buf for stage 2 parsing */ - uint8_t *current_string_buf_loc; - - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - simdjson_inline void start_container(json_iterator &iter) noexcept; - simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder_impl - -// Thin, non-templated entry so each architecture's stage2() keeps calling -// tape_builder::parse_document unchanged. It chooses the bounds-safe -// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the -// choice is a compile-time constant inside the walk: the padded path carries no -// extra branch or load (see tape_builder_impl::visit_string). -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } - - // The stage 2 entry points of dom::document_stream, which picks one per batch - // (see internal::dom_parser_implementation::stage2_next_function). Out of - // line, so that stage2_next() can share the code. - simdjson_warn_unused static simdjson_never_inline error_code parse_next_document( - internal::dom_parser_implementation &dom_parser, dom::document &doc) noexcept { - return parse_document(static_cast(dom_parser), doc); - } - - // A document of a stream batch that stage 2 must bound to its structural - // indexes (see internal::dom_parser_implementation::_bounded_batch). - simdjson_warn_unused static simdjson_never_inline error_code parse_next_bounded_document( - internal::dom_parser_implementation &_dom_parser, dom::document &doc) noexcept { - dom_parser_implementation &dom_parser = static_cast(_dom_parser); - dom_parser.doc = &doc; - json_iterator iter(dom_parser, dom_parser.next_structural_index); - if (dom_parser._unpadded) { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } else { - tape_builder_impl builder(doc); - return iter.walk_document(builder); - } - } -}; - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_root_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { - return iter.visit_primitive(*this, value); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { - return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { - start_container(iter); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { - return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { - constexpr uint32_t start_tape_index = 0; - tape.append(start_tape_index, internal::tape_type::ROOT); - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); - return SUCCESS; -} -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { - return visit_string(iter, key, true); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 - return SUCCESS; -} - -template -simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { - iter.log_value(key ? "key" : "string"); - uint8_t *dst = on_start_string(iter); - // We do not allow replacement when the escape characters are invalid. - // UNPADDED is a compile-time constant chosen once per document by - // tape_builder::parse_document, so the padded build instantiates only the - // plain parse_string call below -- no runtime branch and no flag load. - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); - } else { - dst = stringparsing::parse_string(value+1, dst, false); - } - if (dst == nullptr) { - iter.log_error("Invalid escape in string"); - return STRING_ERROR; - } - on_end_string(dst); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { - return visit_string(iter, value); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("number"); - const uint8_t *num = value; - std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used - SIMDJSON_IF_CONSTEXPR (UNPADDED) { - // numberparsing reads ahead in 8-byte blocks for floats - // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a - // number whose digits reach the final bytes of an unpadded buffer would read - // past it. *(next_structural) is the offset of the token following this - // number, hence an upper bound on where the digits end; when that is within - // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail - // (mirroring visit_root_number). This fires only for numbers near the end. - if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { - const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document - copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, rl); - std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); - num = copy.get(); - } - } - error_code err = numberparsing::parse_number(num, tape); - if (simdjson_unlikely(err == BIGINT_ERROR && - iter.dom_parser._number_as_string)) { - // Write big integer to string buffer using the same format as strings. - // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = num; - if (*p == '-') p++; - while (numberparsing::is_digit(*p)) p++; - // The digit run must be terminated by a structural or whitespace character; otherwise the - // token is malformed (e.g. "123456789123456789123x"). - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - num); - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); - uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, num, len); - dst += len; - on_end_string(dst); - return SUCCESS; - } - return err; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { - // - // We need to make a copy to make sure that the string is space terminated. - // This is not about padding the input, which should already padded up - // to len + SIMDJSON_PADDING. However, we have no control at this stage - // on how the padding was done. What if the input string was padded with nulls? - // It is quite common for an input string to have an extra null character (C string). - // We do not want to allow 9\0 (where \0 is the null character) inside a JSON - // document, but the string "9\0" by itself is fine. So we make a copy and - // pad the input with spaces when we know that there is just one input element. - // This copy is relatively expensive, but it will almost never be called in - // practice unless you are in the strange scenario where you have many JSON - // documents made of single atoms. - // - // In a stream, the input goes on with other documents: copy up to the next - // structural only, not to the end of the batch. - const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); - std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, len); - std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); - error_code error = visit_number(iter, copy.get()); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated - // token at the very end of an unpadded buffer would over-read. Use the - // length-aware form there (the root variant already does this). - const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) - : atomparsing::is_valid_true_atom(value); - if (!ok) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) - : atomparsing::is_valid_false_atom(value); - if (!ok) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) - : atomparsing::is_valid_null_atom(value); - if (!ok) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; -} - -#if SIMDJSON_ENABLE_NAN_INF -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - // For unpadded input use the length-aware validator so the 'infinity'-style - // 8-byte compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) - : atomparsing::is_valid_nan_atom(value); - if (!ok) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { - iter.log_value("nan"); - if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } - tape.append_double(std::numeric_limits::quiet_NaN()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. - // For unpadded input use the length-aware validator so the 'infinity' 8-byte - // compare cannot read past the buffer on a malformed token at the end. - const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) - : atomparsing::is_valid_inf_atom(value); - if (!ok) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { - iter.log_value("inf"); - // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure - if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } - tape.append_double(std::numeric_limits::infinity()); - return SUCCESS; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -// private: - -template -simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { - return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - auto start_index = next_tape_index(iter); - tape.append(start_index+2, start); - tape.append(start_index, end); - return SUCCESS; -} - -template -simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { - iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); - iter.dom_parser.open_containers[iter.depth].count = 0; - tape.skip(); // We don't actually *write* the start element until the end. -} - -template -simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { - // Write the ending tape element, pointing at the start location - const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; - tape.append(start_tape_index, end); - // Write the start tape element, pointing at the end location (and including count) - // count can overflow if it exceeds 24 bits... so we saturate - // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). - const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; - const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); - return SUCCESS; -} - -template -simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { - // we advance the point, accounting for the fact that we have a NULL termination - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); - return current_string_buf_loc + sizeof(uint32_t); -} - -template -simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { - uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); - // TODO check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); - // NULL termination is still handy if you expect all your strings to - // be NULL terminated? It comes at a small cost - *dst = 0; - current_string_buf_loc = dst + 1; -} - -} // namespace stage2 -} // unnamed namespace -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H -/* end file generic/stage2/tape_builder.h for fallback */ - -// -// Stage 1 -// - -namespace simdjson { -namespace fallback { - -simdjson_warn_unused error_code implementation::create_dom_parser_implementation( - size_t capacity, - size_t max_depth, - std::unique_ptr& dst -) const noexcept { - dst.reset( new (std::nothrow) fallback::dom_parser_implementation() ); - if (!dst) { return MEMALLOC; } - if (auto err = dst->set_capacity(capacity)) - return err; - if (auto err = dst->set_max_depth(max_depth)) - return err; - return SUCCESS; -} - -namespace { -namespace stage1 { - -class structural_scanner { -public: - -simdjson_inline structural_scanner(dom_parser_implementation &_parser, stage1_mode _partial) - : buf{_parser.buf}, - next_structural_index{_parser.structural_indexes.get()}, - parser{_parser}, - len{static_cast(_parser.len)}, - partial{_partial} { -} - -simdjson_inline void add_structural() { - *next_structural_index = idx; - next_structural_index++; -} - -simdjson_inline bool is_continuation(uint8_t c) { - return (c & 0xc0) == 0x80; -} - -simdjson_inline void validate_utf8_character() { - // Continuation - if (simdjson_unlikely((buf[idx] & 0x40) == 0)) { - // extra continuation - error = UTF8_ERROR; - idx++; - return; - } - - // 2-byte - if ((buf[idx] & 0x20) == 0) { - // missing continuation - if (simdjson_unlikely(idx+1 >= len || !is_continuation(buf[idx+1]))) { - if (idx+1 >= len && is_streaming(partial)) { idx = len; return; } - error = UTF8_ERROR; - idx++; - return; - } - // overlong: 1100000_ 10______ - if (buf[idx] <= 0xc1) { error = UTF8_ERROR; } - idx += 2; - return; - } - - // 3-byte - if ((buf[idx] & 0x10) == 0) { - // missing continuation - if (simdjson_unlikely(idx+2 >= len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]))) { - if (idx+2 >= len && is_streaming(partial)) { idx = len; return; } - error = UTF8_ERROR; - idx++; - return; - } - // overlong: 11100000 100_____ ________ - if (buf[idx] == 0xe0 && buf[idx+1] <= 0x9f) { error = UTF8_ERROR; } - // surrogates: U+D800-U+DFFF 11101101 101_____ - if (buf[idx] == 0xed && buf[idx+1] >= 0xa0) { error = UTF8_ERROR; } - idx += 3; - return; - } - - // 4-byte - // missing continuation - if (simdjson_unlikely(idx+3 >= len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]) || !is_continuation(buf[idx+3]))) { - if (idx+3 >= len && is_streaming(partial)) { idx = len; return; } - error = UTF8_ERROR; - idx++; - return; - } - // overlong: 11110000 1000____ ________ ________ - if (buf[idx] == 0xf0 && buf[idx+1] <= 0x8f) { error = UTF8_ERROR; } - // too large: > U+10FFFF: - // 11110100 (1001|101_)____ - // 1111(1___|011_|0101) 10______ - // also includes 5, 6, 7 and 8 byte characters: - // 11111___ - if (buf[idx] == 0xf4 && buf[idx+1] >= 0x90) { error = UTF8_ERROR; } - if (buf[idx] >= 0xf5) { error = UTF8_ERROR; } - idx += 4; -} - -static const uint8_t CHAR_TYPE_SPACE = 1 << 0; -static const uint8_t CHAR_TYPE_OPERATOR = 1 << 1; -static const uint8_t CHAR_TYPE_ESC_ASCII = 1 << 2; -static const uint8_t CHAR_TYPE_NON_ASCII = 1 << 3; - -const uint8_t char_table[256] = { - 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, - 0x04, 0x05, 0x05, 0x04, 0x04, 0x05, 0x04, 0x04, - 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, - 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, - 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x02, 0x04, 0x02, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x02, 0x00, 0x02, 0x00, 0x00, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, - 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08 -}; - -simdjson_inline bool char_is_type(uint8_t c, uint8_t type) { - return (char_table[c] & type); -} - -simdjson_inline bool char_is_space(uint8_t c) { - return char_is_type(c, CHAR_TYPE_SPACE); -} - -simdjson_inline bool char_is_operator(uint8_t c) { - return char_is_type(c, CHAR_TYPE_OPERATOR); -} - -simdjson_inline bool char_is_space_or_operator(uint8_t c) { - return char_is_type(c, CHAR_TYPE_SPACE | CHAR_TYPE_OPERATOR); -} - -simdjson_inline bool char_is_ascii_stop(uint8_t c) { - return char_is_type(c, CHAR_TYPE_ESC_ASCII | CHAR_TYPE_NON_ASCII); -} - -// Returns true if the string is unclosed. -simdjson_inline bool validate_string() { - idx++; // skip first quote - while (idx < len) { - do { - if (char_is_ascii_stop(buf[idx])) { break; } - idx++; - } while (idx < len); - if (idx >= len) { return true; } - if (buf[idx] == '"') { - return false; - } - if (buf[idx] == '\\') { - idx += 2; - } else if (simdjson_unlikely(buf[idx] & 0x80)) { - validate_utf8_character(); - } else { - if (buf[idx] < 0x20) { error = UNESCAPED_CHARS; } - idx++; - } - } - if (idx >= len) { return true; } - return false; -} - -// -// Parse the entire input in STEP_SIZE-byte chunks. -// -simdjson_warn_unused simdjson_inline error_code scan() { - bool unclosed_string = false; - for (;idx= len) { break; } - // String - if (buf[idx] == '"') { - add_structural(); - unclosed_string |= validate_string(); - // Operator - } else if (char_is_operator(buf[idx])) { - add_structural(); - // Primitive or invalid character (invalid characters will be checked in stage 2) - } else { - // Anything else, add the structural and go until we find the next one. - // We also stop on RS (0x1E) so that RFC 7464 json_sequence inputs - // like `\x1e"a"\x1e"b"` produce a separate structural for each RS - // rather than being absorbed into a single primitive run, and on '"' - // so that an unclosed string still reaches validate_string(). Neither - // can occur inside a valid primitive. - add_structural(); - while (idx+1 0) { - // The batch holds a single document, whose brackets balance but which - // cannot be valid JSON: it is not too big, it is invalid. - if (unfinished && find_next_document_index(parser, partial_scalar) != 0) { return TAPE_ERROR; } - if(parser.structural_indexes[0] == 0) { - // If the buffer is partial and we started at index 0 but the document is - // incomplete, it's too big to parse. - return CAPACITY; - } else { - // It is possible that the document could be parsed, we just had a lot - // of white space. - parser.n_structural_indexes = 0; - return EMPTY; - } - } - parser.n_structural_indexes = new_structural_indexes; - } else if(partial == stage1_mode::streaming_final) { - if(unclosed_string) { parser.n_structural_indexes--; } - // We truncate the input to the end of the last complete document (or zero). - // Because partial == stage1_mode::streaming_final, it means that we may - // silently ignore trailing garbage. Though it sounds bad, we do it - // deliberately because many people who have streams of JSON documents - // will truncate them for processing. E.g., imagine that you are uncompressing - // the data from a size file or receiving it in chunks from the network. You - // may not know where exactly the last document will be. Meanwhile the - // document_stream instances allow people to know the JSON documents they are - // parsing (see the iterator.source() method). - parser.n_structural_indexes = find_next_document_index(parser); - // We store the initial n_structural_indexes so that the client can see - // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, - // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, - // otherwise, it will copy some prior index. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - // This next line is critical, do not change it unless you understand what you are - // doing. - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (parser.n_structural_indexes == 0) { return EMPTY; } - } else if (partial == stage1_mode::json_sequence_partial) { - // RFC 7464: use RS positions for batch boundaries - // A discarded unclosed string also caps how far the filter may scan. - size_t scan_len = len; - if(unclosed_string) { - parser.n_structural_indexes--; - if (simdjson_unlikely(parser.n_structural_indexes == 0)) { return CAPACITY; } - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - // Repeat the RS that starts the next batch, as in json_structural_indexer.h: - // an invalid last document may make the On-Demand API skip it as a value. - parser.structural_indexes[parser.n_structural_indexes + 1] = next_batch_start; - } else if (partial == stage1_mode::json_sequence_final) { - // RFC 7464: final batch, last document extends to EOF - // As above: a discarded unclosed string caps how far the filter may scan. - size_t scan_len = len; - if(unclosed_string) { - parser.n_structural_indexes--; - scan_len = parser.structural_indexes[parser.n_structural_indexes]; - } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); - // The filter compacted structural_indexes in place and restored the EOF - // sentinel past the compacted end, so the copy below is either the start - // of a truncated document or len, as in streaming_final. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0)) { return EMPTY; } - } else if (partial == stage1_mode::comma_delimited_partial) { - // Comma-delimited: filter root-level commas, use comma positions for batch boundaries - if(unclosed_string) { - parser.n_structural_indexes--; - if (simdjson_unlikely(parser.n_structural_indexes == 0)) { return CAPACITY; } - } - uint32_t next_batch_start = uint32_t(len); - auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); - if (new_structural_indexes == DOCUMENT_TOO_LARGE) { - return CAPACITY; - } - if (new_structural_indexes == 0) { - // An EMPTY batch must still advance next_batch_start, or document_stream - // re-parses the same bytes forever. CAPACITY when it cannot. - if (next_batch_start == 0) { return CAPACITY; } - parser.n_structural_indexes = 0; - parser.structural_indexes[0] = next_batch_start; - return EMPTY; - } - parser.n_structural_indexes = new_structural_indexes; - parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; - } else if (partial == stage1_mode::comma_delimited_final) { - // Comma-delimited: final batch, last document extends to EOF - if(unclosed_string) { parser.n_structural_indexes--; } - uint32_t next_batch_start = uint32_t(len); - parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0)) { return EMPTY; } - } else if(unclosed_string) { error = UNCLOSED_STRING; } - return error; -} - -private: - const uint8_t *buf; - uint32_t *next_structural_index; - dom_parser_implementation &parser; - uint32_t len; - uint32_t idx{0}; - error_code error{SUCCESS}; - stage1_mode partial; -}; // structural_scanner - -} // namespace stage1 -} // unnamed namespace - -simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, stage1_mode partial) noexcept { - this->buf = _buf; - this->len = _len; - stage1::structural_scanner scanner(*this, partial); - return scanner.scan(); -} - -// big table for the minifier -static uint8_t jump_table[256 * 3] = { - 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, - 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, - 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, - 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, - 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, - 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, - 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, - 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, - 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, - 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, - 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, - 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, - 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, - 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, - 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, - 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, - 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, - 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, - 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, - 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, - 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, - 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, - 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, - 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, - 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, - 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, - 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, - 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, - 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, - 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, - 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, -}; - -simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept { - size_t i = 0, pos = 0; - uint8_t quote = 0; - uint8_t nonescape = 1; - - while (i < len) { - unsigned char c = buf[i]; - uint8_t *meta = jump_table + 3 * c; - - quote = quote ^ (meta[0] & nonescape); - dst[pos] = c; - pos += meta[2] | quote; - - i += 1; - nonescape = uint8_t(~nonescape) | (meta[1]); - } - dst_len = pos; // we intentionally do not work with a reference - // for fear of aliasing - return quote ? UNCLOSED_STRING : SUCCESS; -} - -// credit: based on code from Google Fuchsia (Apache Licensed) -simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept { - const uint8_t *data = reinterpret_cast(buf); - uint64_t pos = 0; - uint32_t code_point = 0; - while (pos < len) { - // check of the next 8 bytes are ascii. - uint64_t next_pos = pos + 16; - if (next_pos <= len) { // if it is safe to read 8 more bytes, check that they are ascii - uint64_t v1; - memcpy(&v1, data + pos, sizeof(uint64_t)); - uint64_t v2; - memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); - uint64_t v{v1 | v2}; - if ((v & 0x8080808080808080) == 0) { - pos = next_pos; - continue; - } - } - unsigned char byte = data[pos]; - if (byte < 0x80) { - pos++; - continue; - } else if ((byte & 0xe0) == 0xc0) { - next_pos = pos + 2; - if (next_pos > len) { return false; } - if ((data[pos + 1] & 0xc0) != 0x80) { return false; } - // range check - code_point = (byte & 0x1f) << 6 | (data[pos + 1] & 0x3f); - if (code_point < 0x80 || 0x7ff < code_point) { return false; } - } else if ((byte & 0xf0) == 0xe0) { - next_pos = pos + 3; - if (next_pos > len) { return false; } - if ((data[pos + 1] & 0xc0) != 0x80) { return false; } - if ((data[pos + 2] & 0xc0) != 0x80) { return false; } - // range check - code_point = (byte & 0x0f) << 12 | - (data[pos + 1] & 0x3f) << 6 | - (data[pos + 2] & 0x3f); - if (code_point < 0x800 || 0xffff < code_point || - (0xd7ff < code_point && code_point < 0xe000)) { - return false; - } - } else if ((byte & 0xf8) == 0xf0) { // 0b11110000 - next_pos = pos + 4; - if (next_pos > len) { return false; } - if ((data[pos + 1] & 0xc0) != 0x80) { return false; } - if ((data[pos + 2] & 0xc0) != 0x80) { return false; } - if ((data[pos + 3] & 0xc0) != 0x80) { return false; } - // range check - code_point = - (byte & 0x07) << 18 | (data[pos + 1] & 0x3f) << 12 | - (data[pos + 2] & 0x3f) << 6 | (data[pos + 3] & 0x3f); - if (code_point <= 0xffff || 0x10ffff < code_point) { return false; } - } else { - // we may have a continuation - return false; - } - pos = next_pos; - } - return true; -} - -} // namespace fallback -} // namespace simdjson - -// -// Stage 2 -// - -namespace simdjson { -namespace fallback { - -simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); -} - -simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept { - return stage2::tape_builder::parse_next_document(*this, _doc); -} - -internal::dom_parser_implementation::stage2_next_function_t dom_parser_implementation::stage2_next_function(bool bounded) const noexcept { - return bounded ? stage2::tape_builder::parse_next_bounded_document : stage2::tape_builder::parse_next_document; -} - -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst, bool replacement_char) const noexcept { - return fallback::stringparsing::parse_string(src, dst, replacement_char); -} - -simdjson_warn_unused uint8_t *dom_parser_implementation::parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept { - return fallback::stringparsing::parse_wobbly_string(src, dst); -} - -simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { - auto error = stage1(_buf, _len, stage1_mode::regular); - if (error) { return error; } - return stage2(_doc); -} - -} // namespace fallback -} // namespace simdjson - -/* including simdjson/fallback/end.h: #include */ -/* begin file simdjson/fallback/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* undefining SIMDJSON_IMPLEMENTATION from "fallback" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/fallback/end.h */ - -#endif // SIMDJSON_SRC_FALLBACK_CPP -/* end file fallback.cpp */ -#endif -/* undefining SIMDJSON_CONDITIONAL_INCLUDE */ -#undef SIMDJSON_CONDITIONAL_INCLUDE - -SIMDJSON_POP_DISABLE_UNUSED_WARNINGS - -/* end file simdjson.cpp */ diff --git a/deps/simdjson/simdjson.gyp b/deps/simdjson/simdjson.gyp deleted file mode 100644 index 5c5707a7de28..000000000000 --- a/deps/simdjson/simdjson.gyp +++ /dev/null @@ -1,20 +0,0 @@ -{ - 'variables': { - 'simdjson_sources': [ - 'simdjson.cpp', - ] - }, - 'targets': [ - { - 'target_name': 'simdjson', - 'type': 'static_library', - 'include_dirs': ['.'], - 'direct_dependent_settings': { - 'include_dirs': ['.'], - }, - 'sources': [ - '<@(simdjson_sources)', - ], - }, - ] -} diff --git a/deps/simdjson/simdjson.h b/deps/simdjson/simdjson.h deleted file mode 100644 index 04dbe98eb574..000000000000 --- a/deps/simdjson/simdjson.h +++ /dev/null @@ -1,237888 +0,0 @@ -/* auto-generated on 2026-10-31 19:14:12 -0400. version 5.1.1 Do not edit! */ -/* including simdjson.h: */ -/* begin file simdjson.h */ -#ifndef SIMDJSON_H -#define SIMDJSON_H - -/** - * @mainpage - * - * Check the [README.md](https://github.com/simdjson/simdjson/blob/master/README.md#simdjson--parsing-gigabytes-of-json-per-second). - * - * Sample code. See https://github.com/simdjson/simdjson/blob/master/doc/basics.md for more examples. - - #include "simdjson.h" - - int main(void) { - // load from `twitter.json` file: - simdjson::dom::parser parser; - simdjson::dom::element tweets = parser.load("twitter.json"); - std::cout << tweets["search_metadata"]["count"] << " results." << std::endl; - - // Parse and iterate through an array of objects - auto abstract_json = R"( [ - { "12345" : {"a":12.34, "b":56.78, "c": 9998877} }, - { "12545" : {"a":11.44, "b":12.78, "c": 11111111} } - ] )"_padded; - - for (simdjson::dom::object obj : parser.parse(abstract_json)) { - for(const auto key_value : obj) { - cout << "key: " << key_value.key << " : "; - simdjson::dom::object innerobj = key_value.value; - cout << "a: " << double(innerobj["a"]) << ", "; - cout << "b: " << double(innerobj["b"]) << ", "; - cout << "c: " << int64_t(innerobj["c"]) << endl; - } - } - } - */ - -/* including simdjson/common_defs.h: #include "simdjson/common_defs.h" */ -/* begin file simdjson/common_defs.h */ -#ifndef SIMDJSON_COMMON_DEFS_H -#define SIMDJSON_COMMON_DEFS_H - -#include -/* including simdjson/compiler_check.h: #include "simdjson/compiler_check.h" */ -/* begin file simdjson/compiler_check.h */ -#ifndef SIMDJSON_COMPILER_CHECK_H -#define SIMDJSON_COMPILER_CHECK_H - -#ifndef __cplusplus -#error simdjson requires a C++ compiler -#endif - -#ifndef SIMDJSON_CPLUSPLUS -#if defined(_MSVC_LANG) && !defined(__clang__) -#define SIMDJSON_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG) -#else -#define SIMDJSON_CPLUSPLUS __cplusplus -#endif -#endif - -// C++ 26 -// While C++26 is a working draft, compilers report 202400L in C++26 mode -// (both GCC 16 and Clang 21 do). Update when the standard is finalized. -#if !defined(SIMDJSON_CPLUSPLUS26) && (SIMDJSON_CPLUSPLUS >= 202400L) -#define SIMDJSON_CPLUSPLUS26 1 -#endif - -// C++ 23 -#if !defined(SIMDJSON_CPLUSPLUS23) && (SIMDJSON_CPLUSPLUS >= 202302L) -#define SIMDJSON_CPLUSPLUS23 1 -#endif - -// C++ 20 -#if !defined(SIMDJSON_CPLUSPLUS20) && (SIMDJSON_CPLUSPLUS >= 202002L) -#define SIMDJSON_CPLUSPLUS20 1 -#endif - -// C++ 17 -#if !defined(SIMDJSON_CPLUSPLUS17) && (SIMDJSON_CPLUSPLUS >= 201703L) -#define SIMDJSON_CPLUSPLUS17 1 -#endif - -// C++ 14 -#if !defined(SIMDJSON_CPLUSPLUS14) && (SIMDJSON_CPLUSPLUS >= 201402L) -#define SIMDJSON_CPLUSPLUS14 1 -#endif - -// C++ 11 -#if !defined(SIMDJSON_CPLUSPLUS11) && (SIMDJSON_CPLUSPLUS >= 201103L) -#define SIMDJSON_CPLUSPLUS11 1 -#endif - -#ifndef SIMDJSON_CPLUSPLUS11 -#error simdjson requires a compiler compliant with the C++11 standard -#endif - -#ifndef SIMDJSON_IF_CONSTEXPR -#if SIMDJSON_CPLUSPLUS17 -#define SIMDJSON_IF_CONSTEXPR if constexpr -#else -#define SIMDJSON_IF_CONSTEXPR if -#endif -#endif - -#ifndef SIMDJSON_CONSTEXPR_LAMBDA -#if SIMDJSON_CPLUSPLUS17 -#define SIMDJSON_CONSTEXPR_LAMBDA constexpr -#else -#define SIMDJSON_CONSTEXPR_LAMBDA -#endif -#endif - - - -#ifdef __has_include -#if __has_include() -#include -#endif -#endif - -// Static reflection. -// -// The reflection-based APIs (simdjson::to, document::get, the builder, -// compile-time JSON, annotations) need considerably more than the reflection -// operator. We turn them on only when the compiler advertises all of: -// -// P2996 reflection (^^, splicers, ) __cpp_impl_reflection, -// __cpp_lib_reflection -// P1306 expansion statements (template for) __cpp_expansion_statements -// P3491 std::define_static_string / _array __cpp_lib_define_static -// -// Two further features we rely on have, as of this writing, no feature-test -// macro of their own, so they cannot be checked directly: -// -// P3394 annotations ([[=x]], std::meta::annotations_of) -- used for -// the annotations of simdjson/annotations.h (rename, skip, ...). -// P3289 consteval blocks (consteval { ... }) -- used by compile_time_json. -// -// Every implementation that defines the four macros above also implements -// those two, so requiring the four is sufficient in practice. If that ever -// stops being true, define SIMDJSON_STATIC_REFLECTION=0 to opt out. -// -// SIMDJSON_STATIC_REFLECTION may always be defined by the user (or by the -// build system) to 0 or 1 to override the detection. -// -// Note that C++26 mode alone is not enough: GCC 16 requires -freflection, -// and only then does it define __cpp_impl_reflection. -#ifndef SIMDJSON_STATIC_REFLECTION -#if defined(SIMDJSON_CPLUSPLUS26) && \ - defined(__cpp_impl_reflection) && __cpp_impl_reflection >= 202506L && \ - defined(__cpp_lib_reflection) && __cpp_lib_reflection >= 202506L && \ - defined(__cpp_expansion_statements) && \ - __cpp_expansion_statements >= 202506L && \ - defined(__cpp_lib_define_static) && __cpp_lib_define_static >= 202506L -// __cpp_lib_reflection is the feature-test macro for , so there is no -// need for a separate __has_include check (which would have to be guarded for -// compilers that lack __has_include). -#define SIMDJSON_STATIC_REFLECTION 1 -#else -#define SIMDJSON_STATIC_REFLECTION 0 -#endif -#endif // SIMDJSON_STATIC_REFLECTION - -#if defined(__apple_build_version__) -#if __apple_build_version__ < 14000000 -#define SIMDJSON_CONCEPT_DISABLED 1 // apple-clang/13 doesn't support std::convertible_to -#endif -#endif - -#if defined(__cpp_lib_ranges) && __cpp_lib_ranges >= 201911L -#include -#define SIMDJSON_SUPPORTS_RANGES 1 -#else -#define SIMDJSON_SUPPORTS_RANGES 0 -#endif - -#if defined(__cpp_concepts) && !defined(SIMDJSON_CONCEPT_DISABLED) -#if __cpp_concepts >= 201907L -#include -#define SIMDJSON_SUPPORTS_CONCEPTS 1 -#else -#define SIMDJSON_SUPPORTS_CONCEPTS 0 -#endif -#else // defined(__cpp_concepts) && !defined(SIMDJSON_CONCEPT_DISABLED) -#define SIMDJSON_SUPPORTS_CONCEPTS 0 -#endif // defined(__cpp_concepts) && !defined(SIMDJSON_CONCEPT_DISABLED) - -// copy SIMDJSON_SUPPORTS_CONCEPTS to SIMDJSON_SUPPORTS_DESERIALIZATION. -#if SIMDJSON_SUPPORTS_CONCEPTS -#define SIMDJSON_SUPPORTS_DESERIALIZATION 1 -#else -#define SIMDJSON_SUPPORTS_DESERIALIZATION 0 -#endif - -// The C++20 char8_t type (and std::u8string/std::u8string_view) is available. -// Because all strings that simdjson produces are valid UTF-8, we can offer -// char8_t variants of our string accessors when this macro is set. -#if !defined(SIMDJSON_SUPPORTS_CHAR8_T) -#if defined(__cpp_char8_t) && __cpp_char8_t >= 201811L -#define SIMDJSON_SUPPORTS_CHAR8_T 1 -#else -#define SIMDJSON_SUPPORTS_CHAR8_T 0 -#endif -#endif // !defined(SIMDJSON_SUPPORTS_CHAR8_T) - -// The C++23 fixed-width floating-point types std::float32_t and std::float64_t -// () are available. They are optional even in C++23: a compiler that -// provides them predefines __STDCPP_FLOAT32_T__ and __STDCPP_FLOAT64_T__. -// When these macros are set, we offer get_float32() and get_float64(). -#if !defined(SIMDJSON_SUPPORTS_FLOAT32_T) -#if defined(__STDCPP_FLOAT32_T__) && defined(__has_include) -#if __has_include() -#define SIMDJSON_SUPPORTS_FLOAT32_T 1 -#endif -#endif -#ifndef SIMDJSON_SUPPORTS_FLOAT32_T -#define SIMDJSON_SUPPORTS_FLOAT32_T 0 -#endif -#endif // !defined(SIMDJSON_SUPPORTS_FLOAT32_T) - -#if !defined(SIMDJSON_SUPPORTS_FLOAT64_T) -#if defined(__STDCPP_FLOAT64_T__) && defined(__has_include) -#if __has_include() -#define SIMDJSON_SUPPORTS_FLOAT64_T 1 -#endif -#endif -#ifndef SIMDJSON_SUPPORTS_FLOAT64_T -#define SIMDJSON_SUPPORTS_FLOAT64_T 0 -#endif -#endif // !defined(SIMDJSON_SUPPORTS_FLOAT64_T) - -#if SIMDJSON_SUPPORTS_FLOAT32_T || SIMDJSON_SUPPORTS_FLOAT64_T -#include -#endif - - -#if !defined(SIMDJSON_CONSTEVAL) -#if defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L -#define SIMDJSON_CONSTEVAL 1 -#else -#define SIMDJSON_CONSTEVAL 0 -#endif // defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L -#endif // !defined(SIMDJSON_CONSTEVAL) - -// SIMDJSON_CONSTEXPR_STRING is 'constexpr' when the standard library supports -// constexpr std::string (e.g., libstdc++ 12 or better), and empty otherwise. It -// lets functions that build a std::string be constant expressions when possible -// while still compiling against older standard libraries. -#if !defined(SIMDJSON_CONSTEXPR_STRING) -#if defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L -#define SIMDJSON_CONSTEXPR_STRING constexpr -#else -#define SIMDJSON_CONSTEXPR_STRING -#endif // defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L -#endif // !defined(SIMDJSON_CONSTEXPR_STRING) -#endif // SIMDJSON_COMPILER_CHECK_H -/* end file simdjson/compiler_check.h */ -/* including simdjson/portability.h: #include "simdjson/portability.h" */ -/* begin file simdjson/portability.h */ -#ifndef SIMDJSON_PORTABILITY_H -#define SIMDJSON_PORTABILITY_H - -#include -#include -#include -#include -#include -#include -#ifndef _WIN32 -// strcasecmp, strncasecmp -#include -#endif - -static_assert(CHAR_BIT == 8, "simdjson requires 8-bit bytes"); - - -// We are using size_t without namespace std:: throughout the project -using std::size_t; - -#ifdef _MSC_VER -#define SIMDJSON_VISUAL_STUDIO 1 -/** - * We want to differentiate carefully between - * clang under visual studio and regular visual - * studio. - * - * Under clang for Windows, we enable: - * * target pragmas so that part and only part of the - * code gets compiled for advanced instructions. - * - */ -#ifdef __clang__ -// clang under visual studio -#define SIMDJSON_CLANG_VISUAL_STUDIO 1 -#else -// just regular visual studio (best guess) -#define SIMDJSON_REGULAR_VISUAL_STUDIO 1 -#endif // __clang__ -#endif // _MSC_VER - -#if (defined(__x86_64__) || defined(_M_AMD64)) && !defined(_M_ARM64EC) -#define SIMDJSON_IS_X86_64 1 -#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) -#define SIMDJSON_IS_ARM64 1 -#elif defined(__riscv) && __riscv_xlen == 64 -#define SIMDJSON_IS_RISCV64 1 - - #if __riscv_v_intrinsic >= 11000 - #define SIMDJSON_HAS_RVV_INTRINSICS 1 - #endif - - #if SIMDJSON_HAS_RVV_INTRINSICS && __riscv_vector && __riscv_v_min_vlen >= 128 && __riscv_v_elen >= 64 - #define SIMDJSON_IS_RVV 1 // RISC-V V extension - #endif - - // current toolchains don't support fixed-size SIMD types that don't match VLEN directly - #if __riscv_v_fixed_vlen >= 128 && __riscv_v_fixed_vlen <= 512 - #define SIMDJSON_IS_RVV_VLS 1 - #endif - -#elif defined(__loongarch_lp64) -#define SIMDJSON_IS_LOONGARCH64 1 -#if defined(__loongarch_sx) && defined(__loongarch_asx) - #define SIMDJSON_IS_LSX 1 - #define SIMDJSON_IS_LASX 1 // We can always run both -#elif defined(__loongarch_sx) - #define SIMDJSON_IS_LSX 1 - -// Adjust for runtime dispatching support. -#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) && !defined(__NVCOMPILER) -#if __GNUC__ > 15 || (__GNUC__ == 15 && __GNUC_MINOR__ >= 0) - // We are ok, we will support runtime dispatch for LASX. -#else - // We disable runtime dispatch for LASX, which means that we will not be able to use LASX - // even if it is supported by the hardware. - // Loongson users should update to GCC 15 or better. - #define SIMDJSON_IMPLEMENTATION_LASX 0 -#endif -#else - // We are not using GCC, so we assume that we can support runtime dispatch for LASX. - // https://godbolt.org/z/jcMnrjYhs - #define SIMDJSON_IMPLEMENTATION_LASX 0 -#endif - - - -#endif -#elif defined(__PPC64__) || defined(_M_PPC64) -#define SIMDJSON_IS_PPC64 1 -#if defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) -#define SIMDJSON_IS_PPC64_VMX 1 -#endif // defined(__ALTIVEC__) -#else -#define SIMDJSON_IS_32BITS 1 - -#if defined(_M_IX86) || defined(__i386__) -#define SIMDJSON_IS_X86_32BITS 1 -#elif defined(__arm__) || defined(_M_ARM) -#define SIMDJSON_IS_ARM_32BITS 1 -#elif defined(__PPC__) || defined(_M_PPC) -#define SIMDJSON_IS_PPC_32BITS 1 -#endif - -#endif // defined(__x86_64__) || defined(_M_AMD64) -#ifndef SIMDJSON_IS_32BITS -#define SIMDJSON_IS_32BITS 0 -#endif - -#if SIMDJSON_IS_32BITS -#ifndef SIMDJSON_NO_PORTABILITY_WARNING -// In the future, we should allow programmers -// to get warning. -#endif // SIMDJSON_NO_PORTABILITY_WARNING -#endif // SIMDJSON_IS_32BITS - -#define SIMDJSON_CAT_IMPLEMENTATION_(a,...) a ## __VA_ARGS__ -#define SIMDJSON_CAT(a,...) SIMDJSON_CAT_IMPLEMENTATION_(a, __VA_ARGS__) - -#define SIMDJSON_STRINGIFY_IMPLEMENTATION_(a,...) #a SIMDJSON_STRINGIFY(__VA_ARGS__) -#define SIMDJSON_STRINGIFY(a,...) SIMDJSON_CAT_IMPLEMENTATION_(a, __VA_ARGS__) - -// this is almost standard? -#undef SIMDJSON_STRINGIFY_IMPLEMENTATION_ -#undef SIMDJSON_STRINGIFY -#define SIMDJSON_STRINGIFY_IMPLEMENTATION_(a) #a -#define SIMDJSON_STRINGIFY(a) SIMDJSON_STRINGIFY_IMPLEMENTATION_(a) - -// Our fast kernels require 64-bit systems. -// -// On 32-bit x86, we lack 64-bit popcnt, lzcnt, blsr instructions. -// Furthermore, the number of SIMD registers is reduced. -// -// On 32-bit ARM, we would have smaller registers. -// -// The simdjson users should still have the fallback kernel. It is -// slower, but it should run everywhere. - -// -// Enable valid runtime implementations, and select SIMDJSON_BUILTIN_IMPLEMENTATION -// - -// We are going to use runtime dispatch. -#if defined(SIMDJSON_IS_X86_64) || defined(SIMDJSON_IS_LSX) -#ifdef __clang__ -// clang does not have GCC push pop -// warning: clang attribute push can't be used within a namespace in clang up -// til 8.0 so SIMDJSON_TARGET_REGION and SIMDJSON_UNTARGET_REGION must be *outside* of a -// namespace. -#define SIMDJSON_TARGET_REGION(T) \ - _Pragma(SIMDJSON_STRINGIFY( \ - clang attribute push(__attribute__((target(T))), apply_to = function))) -#define SIMDJSON_UNTARGET_REGION _Pragma("clang attribute pop") -#elif defined(__GNUC__) -// GCC is easier -#define SIMDJSON_TARGET_REGION(T) \ - _Pragma("GCC push_options") _Pragma(SIMDJSON_STRINGIFY(GCC target(T))) -#define SIMDJSON_UNTARGET_REGION _Pragma("GCC pop_options") -#endif // clang then gcc - -#endif // defined(SIMDJSON_IS_X86_64) || defined(SIMDJSON_IS_LSX) - -// Default target region macros don't do anything. -#ifndef SIMDJSON_TARGET_REGION -#define SIMDJSON_TARGET_REGION(T) -#define SIMDJSON_UNTARGET_REGION -#endif - -// Is threading enabled? -#if defined(_REENTRANT) || defined(_MT) -#ifndef SIMDJSON_THREADS_ENABLED -#define SIMDJSON_THREADS_ENABLED -#endif -#endif - -// workaround for large stack sizes under -O0. -// https://github.com/simdjson/simdjson/issues/691 -#ifdef __APPLE__ -#ifndef __OPTIMIZE__ -// Apple systems have small stack sizes in secondary threads. -// Lack of compiler optimization may generate high stack usage. -// Users may want to disable threads for safety, but only when -// in debug mode which we detect by the fact that the __OPTIMIZE__ -// macro is not defined. -#undef SIMDJSON_THREADS_ENABLED -#endif -#endif - - -#if defined(__clang__) -#define SIMDJSON_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined"))) -#elif defined(__GNUC__) -#define SIMDJSON_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined)) -#else -#define SIMDJSON_NO_SANITIZE_UNDEFINED -#endif - -#if defined(__clang__) || defined(__GNUC__) -#define simdjson_pure [[gnu::pure]] -#else -#define simdjson_pure -#endif - -#if defined(__clang__) || defined(__GNUC__) -#if defined(__has_feature) -# if __has_feature(memory_sanitizer) -#define SIMDJSON_NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory"))) -# endif // if __has_feature(memory_sanitizer) -#endif // defined(__has_feature) -#endif -// make sure it is defined as 'nothing' if it is unapplicable. -#ifndef SIMDJSON_NO_SANITIZE_MEMORY -#define SIMDJSON_NO_SANITIZE_MEMORY -#endif - -#if SIMDJSON_VISUAL_STUDIO -// This is one case where we do not distinguish between -// regular visual studio and clang under visual studio. -// clang under Windows has _stricmp (like visual studio) but not strcasecmp (as clang normally has) -#define simdjson_strcasecmp _stricmp -#define simdjson_strncasecmp _strnicmp -#else -// The strcasecmp, strncasecmp, and strcasestr functions do not work with multibyte strings (e.g. UTF-8). -// So they are only useful for ASCII in our context. -// https://www.gnu.org/software/libunistring/manual/libunistring.html#char-_002a-strings -#define simdjson_strcasecmp strcasecmp -#define simdjson_strncasecmp strncasecmp -#endif - -#if (defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG))) && !SIMDJSON_DEVELOPMENT_CHECKS -// If SIMDJSON_DEVELOPMENT_CHECKS is undefined or 0, we consider that we are in release mode. -// If NDEBUG is set, or __OPTIMIZE__ is set, or we are under MSVC in release mode, -// then do away with asserts and use __assume. -// We still recommend that our users set NDEBUG in release mode. -#if SIMDJSON_VISUAL_STUDIO -#define SIMDJSON_UNREACHABLE() __assume(0) -#define SIMDJSON_ASSUME(COND) __assume(COND) -#else -#define SIMDJSON_UNREACHABLE() __builtin_unreachable(); -#define SIMDJSON_ASSUME(COND) do { if (!(COND)) __builtin_unreachable(); } while (0) -#endif - -#else // defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG)) && !SIMDJSON_DEVELOPMENT_CHECKS -// This should only ever be enabled in debug mode. -#define SIMDJSON_UNREACHABLE() assert(0); -#define SIMDJSON_ASSUME(COND) assert(COND) - -#endif - - - -#if defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__ -#define SIMDJSON_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) -#elif defined _WIN32 -#define SIMDJSON_IS_BIG_ENDIAN 0 -#else -#if defined(__APPLE__) || defined(__FreeBSD__) -#include -#elif defined(sun) || defined(__sun) -#include -#elif defined(__MVS__) -#include -#else -#ifdef __has_include -#if __has_include() -#include -#endif //__has_include() -#endif //__has_include -#endif -# -#ifndef __BYTE_ORDER__ -// safe choice -#define SIMDJSON_IS_BIG_ENDIAN 0 -#endif -# -#ifndef __ORDER_LITTLE_ENDIAN__ -// safe choice -#define SIMDJSON_IS_BIG_ENDIAN 0 -#endif -# -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define SIMDJSON_IS_BIG_ENDIAN 0 -#else -#define SIMDJSON_IS_BIG_ENDIAN 1 -#endif -#endif - -#ifndef SIMDJSON_HAS_UNISTD_H -#if defined(__unix__) || defined(__APPLE__) || defined(__linux__) -#define SIMDJSON_HAS_UNISTD_H 1 -#else -#define SIMDJSON_HAS_UNISTD_H 0 -#endif -#endif - -// padded_memory_map availability. -// -// On POSIX platforms the class is always available: the implementation uses -// `mmap` (and a trailing anonymous page for padding) from . -// -// On Windows the class is disabled by default and must be explicitly -// opted into by defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`. Enabling -// it requires: -// 1. `` has been included *before* `` (so that -// this header can see the Win32 types and the `_WINDOWS_` include -// guard), -// 2. the compilation targets Windows 10, version 1803 or later -// (i.e. `NTDDI_VERSION >= NTDDI_WIN10_RS4`, `0x0A000005`). This is -// required because the implementation relies on the modern memory -// APIs introduced with that version (`CreateFileMapping2` / -// `MapViewOfFile3`), -// 3. the link step pulls in an import library that exports those APIs, -// typically `onecore.lib` (or `mincore.lib`). -// -// The `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS` CMake option arranges (1)-(3) -// automatically when building simdjson with its own CMake. Consumers using -// simdjson as a pre-built library are responsible for setting the macro, -// the Windows version macros, and the link library themselves. -// -// If the opt-in conditions are not met on Windows, `padded_memory_map` -// simply does not exist -- any attempt to use it fails at compile time -// with an "unknown identifier" diagnostic rather than silently degrading. -// -// The SIMDJSON_HAS_PADDED_MEMORY_MAP macro reflects whether the class is -// available in the current translation unit. Users may test this macro to -// conditionally compile code that depends on padded_memory_map. -#ifndef SIMDJSON_HAS_PADDED_MEMORY_MAP - #if defined(__unix__) || defined(__APPLE__) || defined(__linux__) - #define SIMDJSON_HAS_PADDED_MEMORY_MAP 1 - #elif defined(_WINDOWS_) && defined(SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS) && SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS - #define SIMDJSON_HAS_PADDED_MEMORY_MAP 1 - #else - #define SIMDJSON_HAS_PADDED_MEMORY_MAP 0 - #endif -#endif - -#endif // SIMDJSON_PORTABILITY_H -/* end file simdjson/portability.h */ -#include - -namespace simdjson { -namespace internal { -/** - * @private - * Scratch capacity that every caller of to_chars must provide. - * - * The emitted decimal is at most ~24 characters, but dragonbox() and - * format_buffer() intentionally write past the logical end with fixed-size - * 16/17-byte memcpy/memset operations so the compiler can inline them (no - * libc mem* dispatch with size-class branches). The extra bytes are required - * for safety of those over-writes; do not shrink this below 40. - * See src/to_chars.cpp and #2805. - */ -// Use an unscoped enum (not static constexpr / inline constexpr): -// - C++11 targets (readme_examples11, quickstart11, ...) still include this header -// - a static constexpr in the amalgamated simdjson.cpp TU is unused there -// (only callers in headers use it) and trips -Wunused-const-variable -Werror -enum : size_t { to_chars_buffer_size = 40 }; -/** - * @private - * Our own implementation of the C++17 to_chars function. - * Defined in src/to_chars - * - * @note The buffer starting at first must have at least to_chars_buffer_size - * bytes of writable storage (see to_chars_buffer_size). - * @note The input number must be finite (NaN/Inf are not supported). - * @note The result is NOT null-terminated. - */ -char *to_chars(char *first, const char *last, double value); -/** - * @private - * A number parsing routine. - * Defined in src/from_chars - */ -double from_chars(const char *first) noexcept; -double from_chars(const char *first, const char* end) noexcept; -/** - * @private - * Same as from_chars, but produces a correctly rounded binary32 (float) value. - * Defined in src/from_chars - */ -float from_chars_float(const char *first) noexcept; -} - -#ifndef SIMDJSON_EXCEPTIONS -#if defined(__cpp_exceptions) || defined(_CPPUNWIND) -#define SIMDJSON_EXCEPTIONS 1 -#else -#define SIMDJSON_EXCEPTIONS 0 -#endif -#endif - -#ifndef SIMDJSON_ENABLE_NAN_INF -#define SIMDJSON_ENABLE_NAN_INF 0 -#endif - -} // namespace simdjson - -#if defined(__GNUC__) - // Marks a block with a name so that MCA analysis can see it. - #define SIMDJSON_BEGIN_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-BEGIN " #name); - #define SIMDJSON_END_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-END " #name); - #define SIMDJSON_DEBUG_BLOCK(name, block) BEGIN_DEBUG_BLOCK(name); block; END_DEBUG_BLOCK(name); -#else - #define SIMDJSON_BEGIN_DEBUG_BLOCK(name) - #define SIMDJSON_END_DEBUG_BLOCK(name) - #define SIMDJSON_DEBUG_BLOCK(name, block) -#endif - -// Align to N-byte boundary -#define SIMDJSON_ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1)) - -#if SIMDJSON_REGULAR_VISUAL_STUDIO - // We could use [[deprecated]] but it requires C++14 - #define simdjson_deprecated __declspec(deprecated) - - #define simdjson_really_inline __forceinline - #define simdjson_never_inline inline __declspec(noinline) - #define simdjson_really_flatten [[msvc::flatten]] - - #define simdjson_unused - #define simdjson_warn_unused - - #ifndef simdjson_likely - #define simdjson_likely(x) x - #endif - #ifndef simdjson_unlikely - #define simdjson_unlikely(x) x - #endif - - #define SIMDJSON_PUSH_DISABLE_WARNINGS __pragma(warning( push )) - #define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS __pragma(warning( push, 0 )) - #define SIMDJSON_DISABLE_VS_WARNING(WARNING_NUMBER) __pragma(warning( disable : WARNING_NUMBER )) - // Get rid of Intellisense-only warnings (Code Analysis) - // Though __has_include is C++17, it is supported in Visual Studio 2017 or better (_MSC_VER>=1910). - #ifdef __has_include - #if __has_include() - #include - #define SIMDJSON_DISABLE_UNDESIRED_WARNINGS SIMDJSON_DISABLE_VS_WARNING(ALL_CPPCORECHECK_WARNINGS) - #endif - #endif - - #ifndef SIMDJSON_DISABLE_UNDESIRED_WARNINGS - #define SIMDJSON_DISABLE_UNDESIRED_WARNINGS - #endif - - #define SIMDJSON_DISABLE_DEPRECATED_WARNING SIMDJSON_DISABLE_VS_WARNING(4996) - #define SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING - #define SIMDJSON_POP_DISABLE_WARNINGS __pragma(warning( pop )) - - #define SIMDJSON_PUSH_DISABLE_UNUSED_WARNINGS - #define SIMDJSON_POP_DISABLE_UNUSED_WARNINGS - -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - // We could use [[deprecated]] but it requires C++14 - #define simdjson_deprecated __attribute__((deprecated)) - - #define simdjson_really_inline inline __attribute__((always_inline)) - #define simdjson_never_inline inline __attribute__((noinline)) - #define simdjson_really_flatten [[gnu::flatten]] - - #define simdjson_unused __attribute__((unused)) - #define simdjson_warn_unused __attribute__((warn_unused_result)) - - #ifndef simdjson_likely - #define simdjson_likely(x) __builtin_expect(!!(x), 1) - #endif - #ifndef simdjson_unlikely - #define simdjson_unlikely(x) __builtin_expect(!!(x), 0) - #endif - - #define SIMDJSON_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push") - // gcc doesn't seem to disable all warnings with all and extra, add warnings here as necessary - // We do it separately for clang since it has different warnings. - #ifdef __clang__ - // clang is missing -Wmaybe-uninitialized. - #define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \ - SIMDJSON_DISABLE_GCC_WARNING(-Weffc++) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wall) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wconversion) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wextra) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wattributes) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wreturn-type) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wshadow) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wunused-parameter) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wunused-variable) - #else // __clang__ - #define SIMDJSON_PUSH_DISABLE_ALL_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \ - SIMDJSON_DISABLE_GCC_WARNING(-Weffc++) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wall) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wconversion) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wextra) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wattributes) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wreturn-type) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wshadow) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wunused-parameter) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wunused-variable) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wmaybe-uninitialized) \ - SIMDJSON_DISABLE_GCC_WARNING(-Wformat-security) - #endif // __clang__ - - #define SIMDJSON_PRAGMA(P) _Pragma(#P) - #define SIMDJSON_DISABLE_GCC_WARNING(WARNING) SIMDJSON_PRAGMA(GCC diagnostic ignored #WARNING) - #if SIMDJSON_CLANG_VISUAL_STUDIO - #define SIMDJSON_DISABLE_UNDESIRED_WARNINGS SIMDJSON_DISABLE_GCC_WARNING(-Wmicrosoft-include) - #else - #define SIMDJSON_DISABLE_UNDESIRED_WARNINGS - #endif - #define SIMDJSON_DISABLE_DEPRECATED_WARNING SIMDJSON_DISABLE_GCC_WARNING(-Wdeprecated-declarations) - #define SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING SIMDJSON_DISABLE_GCC_WARNING(-Wstrict-overflow) - #define SIMDJSON_POP_DISABLE_WARNINGS _Pragma("GCC diagnostic pop") - - #define SIMDJSON_PUSH_DISABLE_UNUSED_WARNINGS SIMDJSON_PUSH_DISABLE_WARNINGS \ - SIMDJSON_DISABLE_GCC_WARNING(-Wunused) - #define SIMDJSON_POP_DISABLE_UNUSED_WARNINGS SIMDJSON_POP_DISABLE_WARNINGS - - - -#endif // MSC_VER - -#if defined(simdjson_inline) - // Prefer the user's definition of simdjson_inline; don't define it ourselves. -#elif defined(__GNUC__) && !defined(__OPTIMIZE__) - // If optimizations are disabled, forcing inlining can lead to significant - // code bloat and high compile times. Don't use simdjson_really_inline for - // unoptimized builds. - #define simdjson_inline inline -#else - // Force inlining for most simdjson functions. - #define simdjson_inline simdjson_really_inline -#endif - -#if defined(simdjson_flatten) - // Prefer the user's definition of simdjson_flatten; don't define it ourselves. -#elif (defined(__GNUC__) && !defined(__OPTIMIZE__)) || (defined(_DEBUG) && _MSC_VER ) - // Flattening can lead to significant code bloat and high compile times. Don't use it for unoptimized builds. - #define simdjson_flatten -#else - #define simdjson_flatten simdjson_really_flatten -#endif - -#if SIMDJSON_VISUAL_STUDIO - /** - * Windows users need to do some extra work when building - * or using a dynamic library (DLL). When building, we need - * to set SIMDJSON_DLLIMPORTEXPORT to __declspec(dllexport). - * When *using* the DLL, the user needs to set - * SIMDJSON_DLLIMPORTEXPORT __declspec(dllimport). - * - * Static libraries not need require such work. - * - * It does not matter here whether you are using - * the regular visual studio or clang under visual - * studio, you still need to handle these issues. - * - * Non-Windows systems do not have this complexity. - */ - #if SIMDJSON_BUILDING_WINDOWS_DYNAMIC_LIBRARY - // We set SIMDJSON_BUILDING_WINDOWS_DYNAMIC_LIBRARY when we build a DLL under Windows. - // It should never happen that both SIMDJSON_BUILDING_WINDOWS_DYNAMIC_LIBRARY and - // SIMDJSON_USING_WINDOWS_DYNAMIC_LIBRARY are set. - #define SIMDJSON_DLLIMPORTEXPORT __declspec(dllexport) - #elif SIMDJSON_USING_WINDOWS_DYNAMIC_LIBRARY - // Windows user who call a dynamic library should set SIMDJSON_USING_WINDOWS_DYNAMIC_LIBRARY to 1. - #define SIMDJSON_DLLIMPORTEXPORT __declspec(dllimport) - #else - // We assume by default static linkage - #define SIMDJSON_DLLIMPORTEXPORT - #endif -#else - #define SIMDJSON_DLLIMPORTEXPORT -#endif - -// C++17 requires string_view. -#if SIMDJSON_CPLUSPLUS17 -#define SIMDJSON_HAS_STRING_VIEW -#include // by the standard, this has to be safe. -#endif - -// This macro (__cpp_lib_string_view) has to be defined -// for C++17 and better, but if it is otherwise defined, -// we are going to assume that string_view is available -// even if we do not have C++17 support. -#ifdef __cpp_lib_string_view -#define SIMDJSON_HAS_STRING_VIEW -#include -#endif - -// Some systems have string_view even if we do not have C++17 support, -// and even if __cpp_lib_string_view is undefined, it is the case -// with Apple clang version 11. -// We must handle it. *This is important.* -#ifndef _MSC_VER -#ifndef SIMDJSON_HAS_STRING_VIEW -#if defined __has_include -// do not combine the next #if with the previous one (unsafe) -#if __has_include () -// now it is safe to trigger the include -#include // though the file is there, it does not follow that we got the implementation -#if defined(_LIBCPP_STRING_VIEW) -// Ah! So we under libc++ which under its Library Fundamentals Technical Specification, which preceded C++17, -// included string_view. -// This means that we have string_view *even though* we may not have C++17. -#define SIMDJSON_HAS_STRING_VIEW -#endif // _LIBCPP_STRING_VIEW -#endif // __has_include () -#endif // defined __has_include -#endif // def SIMDJSON_HAS_STRING_VIEW -#endif // def _MSC_VER -// end of complicated but important routine to try to detect string_view. - -// -// Backfill std::string_view using nonstd::string_view on systems where -// we expect that string_view is missing. Important: if we get this wrong, -// we will end up with two string_view definitions and potential trouble. -// That is why we work so hard above to avoid it. -// -#ifndef SIMDJSON_HAS_STRING_VIEW -SIMDJSON_PUSH_DISABLE_ALL_WARNINGS -/* including simdjson/nonstd/string_view.hpp: #include "simdjson/nonstd/string_view.hpp" */ -/* begin file simdjson/nonstd/string_view.hpp */ -// Copyright 2017-2020 by Martin Moene -// -// string-view lite, a C++17-like string_view for C++98 and later. -// For more information see https://github.com/martinmoene/string-view-lite -// -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - - -#ifndef NONSTD_SV_LITE_H_INCLUDED -#define NONSTD_SV_LITE_H_INCLUDED - -#define string_view_lite_MAJOR 1 -#define string_view_lite_MINOR 8 -#define string_view_lite_PATCH 0 - -#define string_view_lite_VERSION nssv_STRINGIFY(string_view_lite_MAJOR) "." nssv_STRINGIFY(string_view_lite_MINOR) "." nssv_STRINGIFY(string_view_lite_PATCH) - -#define nssv_STRINGIFY( x ) nssv_STRINGIFY_( x ) -#define nssv_STRINGIFY_( x ) #x - -// string-view lite configuration: - -#define nssv_STRING_VIEW_DEFAULT 0 -#define nssv_STRING_VIEW_NONSTD 1 -#define nssv_STRING_VIEW_STD 2 - -// tweak header support: - -#ifdef __has_include -# if __has_include() -# include -# endif -#define nssv_HAVE_TWEAK_HEADER 1 -#else -#define nssv_HAVE_TWEAK_HEADER 0 -//# pragma message("string_view.hpp: Note: Tweak header not supported.") -#endif - -// string_view selection and configuration: - -#if !defined( nssv_CONFIG_SELECT_STRING_VIEW ) -# define nssv_CONFIG_SELECT_STRING_VIEW ( nssv_HAVE_STD_STRING_VIEW ? nssv_STRING_VIEW_STD : nssv_STRING_VIEW_NONSTD ) -#endif - -#ifndef nssv_CONFIG_STD_SV_OPERATOR -# define nssv_CONFIG_STD_SV_OPERATOR 0 -#endif - -#ifndef nssv_CONFIG_USR_SV_OPERATOR -# define nssv_CONFIG_USR_SV_OPERATOR 1 -#endif - -#ifdef nssv_CONFIG_CONVERSION_STD_STRING -# define nssv_CONFIG_CONVERSION_STD_STRING_CLASS_METHODS nssv_CONFIG_CONVERSION_STD_STRING -# define nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS nssv_CONFIG_CONVERSION_STD_STRING -#endif - -#ifndef nssv_CONFIG_CONVERSION_STD_STRING_CLASS_METHODS -# define nssv_CONFIG_CONVERSION_STD_STRING_CLASS_METHODS 1 -#endif - -#ifndef nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS -# define nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS 1 -#endif - -#ifndef nssv_CONFIG_NO_STREAM_INSERTION -# define nssv_CONFIG_NO_STREAM_INSERTION 0 -#endif - -#ifndef nssv_CONFIG_CONSTEXPR11_STD_SEARCH -# define nssv_CONFIG_CONSTEXPR11_STD_SEARCH 1 -#endif - -// Control presence of exception handling (try and auto discover): - -#ifndef nssv_CONFIG_NO_EXCEPTIONS -# if defined(_MSC_VER) -# include // for _HAS_EXCEPTIONS -# endif -# if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || (_HAS_EXCEPTIONS) -# define nssv_CONFIG_NO_EXCEPTIONS 0 -# else -# define nssv_CONFIG_NO_EXCEPTIONS 1 -# endif -#endif - -// C++ language version detection (C++23 is speculative): -// Note: VC14.0/1900 (VS2015) lacks too much from C++14. - -#ifndef nssv_CPLUSPLUS -# if defined(_MSVC_LANG ) && !defined(__clang__) -# define nssv_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG ) -# else -# define nssv_CPLUSPLUS __cplusplus -# endif -#endif - -#define nssv_CPP98_OR_GREATER ( nssv_CPLUSPLUS >= 199711L ) -#define nssv_CPP11_OR_GREATER ( nssv_CPLUSPLUS >= 201103L ) -#define nssv_CPP11_OR_GREATER_ ( nssv_CPLUSPLUS >= 201103L ) -#define nssv_CPP14_OR_GREATER ( nssv_CPLUSPLUS >= 201402L ) -#define nssv_CPP17_OR_GREATER ( nssv_CPLUSPLUS >= 201703L ) -#define nssv_CPP20_OR_GREATER ( nssv_CPLUSPLUS >= 202002L ) -#define nssv_CPP23_OR_GREATER ( nssv_CPLUSPLUS >= 202300L ) - -// use C++17 std::string_view if available and requested: - -#if nssv_CPP17_OR_GREATER && defined(__has_include ) -# if __has_include( ) -# define nssv_HAVE_STD_STRING_VIEW 1 -# else -# define nssv_HAVE_STD_STRING_VIEW 0 -# endif -#else -# define nssv_HAVE_STD_STRING_VIEW 0 -#endif - -#define nssv_USES_STD_STRING_VIEW ( (nssv_CONFIG_SELECT_STRING_VIEW == nssv_STRING_VIEW_STD) || ((nssv_CONFIG_SELECT_STRING_VIEW == nssv_STRING_VIEW_DEFAULT) && nssv_HAVE_STD_STRING_VIEW) ) - -#define nssv_HAVE_STARTS_WITH ( nssv_CPP20_OR_GREATER || !nssv_USES_STD_STRING_VIEW ) -#define nssv_HAVE_ENDS_WITH nssv_HAVE_STARTS_WITH - -// -// Use C++17 std::string_view: -// - -#if nssv_USES_STD_STRING_VIEW - -#include - -// Extensions for std::string: - -#if nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS - -#include - -namespace nonstd { - -template< class CharT, class Traits, class Allocator = std::allocator > -std::basic_string -to_string( std::basic_string_view v, Allocator const & a = Allocator() ) -{ - return std::basic_string( v.begin(), v.end(), a ); -} - -template< class CharT, class Traits, class Allocator > -std::basic_string_view -to_string_view( std::basic_string const & s ) -{ - return std::basic_string_view( s.data(), s.size() ); -} - -// Literal operators sv and _sv: - -#if nssv_CONFIG_STD_SV_OPERATOR - -using namespace std::literals::string_view_literals; - -#endif - -#if nssv_CONFIG_USR_SV_OPERATOR - -inline namespace literals { -inline namespace string_view_literals { - - -constexpr std::string_view operator ""_sv( const char* str, size_t len ) noexcept // (1) -{ - return std::string_view{ str, len }; -} - -constexpr std::u16string_view operator ""_sv( const char16_t* str, size_t len ) noexcept // (2) -{ - return std::u16string_view{ str, len }; -} - -constexpr std::u32string_view operator ""_sv( const char32_t* str, size_t len ) noexcept // (3) -{ - return std::u32string_view{ str, len }; -} - -constexpr std::wstring_view operator ""_sv( const wchar_t* str, size_t len ) noexcept // (4) -{ - return std::wstring_view{ str, len }; -} - -}} // namespace literals::string_view_literals - -#endif // nssv_CONFIG_USR_SV_OPERATOR - -} // namespace nonstd - -#endif // nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS - -namespace nonstd { - -using std::string_view; -using std::wstring_view; -using std::u16string_view; -using std::u32string_view; -using std::basic_string_view; - -// literal "sv" and "_sv", see above - -using std::operator==; -using std::operator!=; -using std::operator<; -using std::operator<=; -using std::operator>; -using std::operator>=; - -using std::operator<<; - -} // namespace nonstd - -#else // nssv_HAVE_STD_STRING_VIEW - -// -// Before C++17: use string_view lite: -// - -// Compiler versions: -// -// MSVC++ 6.0 _MSC_VER == 1200 nssv_COMPILER_MSVC_VERSION == 60 (Visual Studio 6.0) -// MSVC++ 7.0 _MSC_VER == 1300 nssv_COMPILER_MSVC_VERSION == 70 (Visual Studio .NET 2002) -// MSVC++ 7.1 _MSC_VER == 1310 nssv_COMPILER_MSVC_VERSION == 71 (Visual Studio .NET 2003) -// MSVC++ 8.0 _MSC_VER == 1400 nssv_COMPILER_MSVC_VERSION == 80 (Visual Studio 2005) -// MSVC++ 9.0 _MSC_VER == 1500 nssv_COMPILER_MSVC_VERSION == 90 (Visual Studio 2008) -// MSVC++ 10.0 _MSC_VER == 1600 nssv_COMPILER_MSVC_VERSION == 100 (Visual Studio 2010) -// MSVC++ 11.0 _MSC_VER == 1700 nssv_COMPILER_MSVC_VERSION == 110 (Visual Studio 2012) -// MSVC++ 12.0 _MSC_VER == 1800 nssv_COMPILER_MSVC_VERSION == 120 (Visual Studio 2013) -// MSVC++ 14.0 _MSC_VER == 1900 nssv_COMPILER_MSVC_VERSION == 140 (Visual Studio 2015) -// MSVC++ 14.1 _MSC_VER >= 1910 nssv_COMPILER_MSVC_VERSION == 141 (Visual Studio 2017) -// MSVC++ 14.2 _MSC_VER >= 1920 nssv_COMPILER_MSVC_VERSION == 142 (Visual Studio 2019) - -#if defined(_MSC_VER ) && !defined(__clang__) -# define nssv_COMPILER_MSVC_VER (_MSC_VER ) -# define nssv_COMPILER_MSVC_VERSION (_MSC_VER / 10 - 10 * ( 5 + (_MSC_VER < 1900 ) ) ) -#else -# define nssv_COMPILER_MSVC_VER 0 -# define nssv_COMPILER_MSVC_VERSION 0 -#endif - -#define nssv_COMPILER_VERSION( major, minor, patch ) ( 10 * ( 10 * (major) + (minor) ) + (patch) ) - -#if defined( __apple_build_version__ ) -# define nssv_COMPILER_APPLECLANG_VERSION nssv_COMPILER_VERSION(__clang_major__, __clang_minor__, __clang_patchlevel__) -# define nssv_COMPILER_CLANG_VERSION 0 -#elif defined( __clang__ ) -# define nssv_COMPILER_APPLECLANG_VERSION 0 -# define nssv_COMPILER_CLANG_VERSION nssv_COMPILER_VERSION(__clang_major__, __clang_minor__, __clang_patchlevel__) -#else -# define nssv_COMPILER_APPLECLANG_VERSION 0 -# define nssv_COMPILER_CLANG_VERSION 0 -#endif - -#if defined(__GNUC__) && !defined(__clang__) -# define nssv_COMPILER_GNUC_VERSION nssv_COMPILER_VERSION(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) -#else -# define nssv_COMPILER_GNUC_VERSION 0 -#endif - -// half-open range [lo..hi): -#define nssv_BETWEEN( v, lo, hi ) ( (lo) <= (v) && (v) < (hi) ) - -// Presence of language and library features: - -#ifdef _HAS_CPP0X -# define nssv_HAS_CPP0X _HAS_CPP0X -#else -# define nssv_HAS_CPP0X 0 -#endif - -// Unless defined otherwise below, consider VC14 as C++11 for string-view-lite: - -#if nssv_COMPILER_MSVC_VER >= 1900 -# undef nssv_CPP11_OR_GREATER -# define nssv_CPP11_OR_GREATER 1 -#endif - -#define nssv_CPP11_90 (nssv_CPP11_OR_GREATER_ || nssv_COMPILER_MSVC_VER >= 1500) -#define nssv_CPP11_100 (nssv_CPP11_OR_GREATER_ || nssv_COMPILER_MSVC_VER >= 1600) -#define nssv_CPP11_110 (nssv_CPP11_OR_GREATER_ || nssv_COMPILER_MSVC_VER >= 1700) -#define nssv_CPP11_120 (nssv_CPP11_OR_GREATER_ || nssv_COMPILER_MSVC_VER >= 1800) -#define nssv_CPP11_140 (nssv_CPP11_OR_GREATER_ || nssv_COMPILER_MSVC_VER >= 1900) -#define nssv_CPP11_141 (nssv_CPP11_OR_GREATER_ || nssv_COMPILER_MSVC_VER >= 1910) - -#define nssv_CPP14_000 (nssv_CPP14_OR_GREATER) -#define nssv_CPP17_000 (nssv_CPP17_OR_GREATER) - -// Presence of C++11 language features: - -#define nssv_HAVE_CONSTEXPR_11 nssv_CPP11_140 -#define nssv_HAVE_EXPLICIT_CONVERSION nssv_CPP11_140 -#define nssv_HAVE_INLINE_NAMESPACE nssv_CPP11_140 -#define nssv_HAVE_IS_DEFAULT nssv_CPP11_140 -#define nssv_HAVE_IS_DELETE nssv_CPP11_140 -#define nssv_HAVE_NOEXCEPT nssv_CPP11_140 -#define nssv_HAVE_NULLPTR nssv_CPP11_100 -#define nssv_HAVE_REF_QUALIFIER nssv_CPP11_140 -#define nssv_HAVE_UNICODE_LITERALS nssv_CPP11_140 -#define nssv_HAVE_USER_DEFINED_LITERALS nssv_CPP11_140 -#define nssv_HAVE_WCHAR16_T nssv_CPP11_100 -#define nssv_HAVE_WCHAR32_T nssv_CPP11_100 - -#if ! ( ( nssv_CPP11_OR_GREATER && nssv_COMPILER_CLANG_VERSION ) || nssv_BETWEEN( nssv_COMPILER_CLANG_VERSION, 300, 400 ) ) -# define nssv_HAVE_STD_DEFINED_LITERALS nssv_CPP11_140 -#else -# define nssv_HAVE_STD_DEFINED_LITERALS 0 -#endif - -// Presence of C++14 language features: - -#define nssv_HAVE_CONSTEXPR_14 nssv_CPP14_000 - -// Presence of C++17 language features: - -#define nssv_HAVE_NODISCARD nssv_CPP17_000 - -// Presence of C++ library features: - -#define nssv_HAVE_STD_HASH nssv_CPP11_120 - -// Presence of compiler intrinsics: - -// Providing char-type specializations for compare() and length() that -// use compiler intrinsics can improve compile- and run-time performance. -// -// The challenge is in using the right combinations of builtin availability -// and its constexpr-ness. -// -// | compiler | __builtin_memcmp (constexpr) | memcmp (constexpr) | -// |----------|------------------------------|---------------------| -// | clang | 4.0 (>= 4.0 ) | any (? ) | -// | clang-a | 9.0 (>= 9.0 ) | any (? ) | -// | gcc | any (constexpr) | any (? ) | -// | msvc | >= 14.2 C++17 (>= 14.2 ) | any (? ) | - -#define nssv_HAVE_BUILTIN_VER ( (nssv_CPP17_000 && nssv_COMPILER_MSVC_VERSION >= 142) || nssv_COMPILER_GNUC_VERSION > 0 || nssv_COMPILER_CLANG_VERSION >= 400 || nssv_COMPILER_APPLECLANG_VERSION >= 900 ) -#define nssv_HAVE_BUILTIN_CE ( nssv_HAVE_BUILTIN_VER ) - -#define nssv_HAVE_BUILTIN_MEMCMP ( (nssv_HAVE_CONSTEXPR_14 && nssv_HAVE_BUILTIN_CE) || !nssv_HAVE_CONSTEXPR_14 ) -#define nssv_HAVE_BUILTIN_STRLEN ( (nssv_HAVE_CONSTEXPR_11 && nssv_HAVE_BUILTIN_CE) || !nssv_HAVE_CONSTEXPR_11 ) - -#ifdef __has_builtin -# define nssv_HAVE_BUILTIN( x ) __has_builtin( x ) -#else -# define nssv_HAVE_BUILTIN( x ) 0 -#endif - -#if nssv_HAVE_BUILTIN(__builtin_memcmp) || nssv_HAVE_BUILTIN_VER -# define nssv_BUILTIN_MEMCMP __builtin_memcmp -#else -# define nssv_BUILTIN_MEMCMP memcmp -#endif - -#if nssv_HAVE_BUILTIN(__builtin_strlen) || nssv_HAVE_BUILTIN_VER -# define nssv_BUILTIN_STRLEN __builtin_strlen -#else -# define nssv_BUILTIN_STRLEN strlen -#endif - -// C++ feature usage: - -#if nssv_HAVE_CONSTEXPR_11 -# define nssv_constexpr constexpr -#else -# define nssv_constexpr /*constexpr*/ -#endif - -#if nssv_HAVE_CONSTEXPR_14 -# define nssv_constexpr14 constexpr -#else -# define nssv_constexpr14 /*constexpr*/ -#endif - -#if nssv_HAVE_EXPLICIT_CONVERSION -# define nssv_explicit explicit -#else -# define nssv_explicit /*explicit*/ -#endif - -#if nssv_HAVE_INLINE_NAMESPACE -# define nssv_inline_ns inline -#else -# define nssv_inline_ns /*inline*/ -#endif - -#if nssv_HAVE_NOEXCEPT -# define nssv_noexcept noexcept -#else -# define nssv_noexcept /*noexcept*/ -#endif - -//#if nssv_HAVE_REF_QUALIFIER -//# define nssv_ref_qual & -//# define nssv_refref_qual && -//#else -//# define nssv_ref_qual /*&*/ -//# define nssv_refref_qual /*&&*/ -//#endif - -#if nssv_HAVE_NULLPTR -# define nssv_nullptr nullptr -#else -# define nssv_nullptr NULL -#endif - -#if nssv_HAVE_NODISCARD -# define nssv_nodiscard simdjson_warn_unused -#else -# define nssv_nodiscard /*simdjson_warn_unused*/ -#endif - -// Additional includes: - -#include -#include -#include -#include -#include // std::char_traits<> - -#if ! nssv_CONFIG_NO_STREAM_INSERTION -# include -#endif - -#if ! nssv_CONFIG_NO_EXCEPTIONS -# include -#endif - -#if nssv_CPP11_OR_GREATER -# include -#endif - -// Clang, GNUC, MSVC warning suppression macros: - -#if defined(__clang__) -# pragma clang diagnostic ignored "-Wreserved-user-defined-literal" -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wuser-defined-literals" -#elif nssv_COMPILER_GNUC_VERSION >= 480 -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wliteral-suffix" -#endif // __clang__ - -#if nssv_COMPILER_MSVC_VERSION >= 140 -# define nssv_SUPPRESS_MSGSL_WARNING(expr) [[gsl::suppress(expr)]] -# define nssv_SUPPRESS_MSVC_WARNING(code, descr) __pragma(warning(suppress: code) ) -# define nssv_DISABLE_MSVC_WARNINGS(codes) __pragma(warning(push)) __pragma(warning(disable: codes)) -#else -# define nssv_SUPPRESS_MSGSL_WARNING(expr) -# define nssv_SUPPRESS_MSVC_WARNING(code, descr) -# define nssv_DISABLE_MSVC_WARNINGS(codes) -#endif - -#if defined(__clang__) -# define nssv_RESTORE_WARNINGS() _Pragma("clang diagnostic pop") -#elif nssv_COMPILER_GNUC_VERSION >= 480 -# define nssv_RESTORE_WARNINGS() _Pragma("GCC diagnostic pop") -#elif nssv_COMPILER_MSVC_VERSION >= 140 -# define nssv_RESTORE_WARNINGS() __pragma(warning(pop )) -#else -# define nssv_RESTORE_WARNINGS() -#endif - -// Suppress the following MSVC (GSL) warnings: -// - C4455, non-gsl : 'operator ""sv': literal suffix identifiers that do not -// start with an underscore are reserved -// - C26472, gsl::t.1 : don't use a static_cast for arithmetic conversions; -// use brace initialization, gsl::narrow_cast or gsl::narow -// - C26481: gsl::b.1 : don't use pointer arithmetic. Use span instead - -nssv_DISABLE_MSVC_WARNINGS( 4455 26481 26472 ) -//nssv_DISABLE_CLANG_WARNINGS( "-Wuser-defined-literals" ) -//nssv_DISABLE_GNUC_WARNINGS( -Wliteral-suffix ) - -namespace nonstd { namespace sv_lite { - -// -// basic_string_view declaration: -// - -template -< - class CharT, - class Traits = std::char_traits -> -class basic_string_view; - -namespace detail { - -// support constexpr comparison in C++14; -// for C++17 and later, use provided traits: - -template< typename CharT > -inline nssv_constexpr14 int compare( CharT const * s1, CharT const * s2, std::size_t count ) -{ - while ( count-- != 0 ) - { - if ( *s1 < *s2 ) return -1; - if ( *s1 > *s2 ) return +1; - ++s1; ++s2; - } - return 0; -} - -#if nssv_HAVE_BUILTIN_MEMCMP - -// specialization of compare() for char, see also generic compare() above: - -inline nssv_constexpr14 int compare( char const * s1, char const * s2, std::size_t count ) -{ - return nssv_BUILTIN_MEMCMP( s1, s2, count ); -} - -#endif - -#if nssv_HAVE_BUILTIN_STRLEN - -// specialization of length() for char, see also generic length() further below: - -inline nssv_constexpr std::size_t length( char const * s ) -{ - return nssv_BUILTIN_STRLEN( s ); -} - -#endif - -#if defined(__OPTIMIZE__) - -// gcc, clang provide __OPTIMIZE__ -// Expect tail call optimization to make length() non-recursive: - -template< typename CharT > -inline nssv_constexpr std::size_t length( CharT * s, std::size_t result = 0 ) -{ - return *s == '\0' ? result : length( s + 1, result + 1 ); -} - -#else // OPTIMIZE - -// non-recursive: - -template< typename CharT > -inline nssv_constexpr14 std::size_t length( CharT * s ) -{ - std::size_t result = 0; - while ( *s++ != '\0' ) - { - ++result; - } - return result; -} - -#endif // OPTIMIZE - -#if nssv_CPP11_OR_GREATER && ! nssv_CPP17_OR_GREATER -#if defined(__OPTIMIZE__) - -// gcc, clang provide __OPTIMIZE__ -// Expect tail call optimization to make search() non-recursive: - -template< class CharT, class Traits = std::char_traits > -constexpr const CharT* search( basic_string_view haystack, basic_string_view needle ) -{ - return haystack.starts_with( needle ) ? haystack.begin() : - haystack.empty() ? haystack.end() : search( haystack.substr(1), needle ); -} - -#else // OPTIMIZE - -// non-recursive: - -#if nssv_CONFIG_CONSTEXPR11_STD_SEARCH - -template< class CharT, class Traits = std::char_traits > -constexpr const CharT* search( basic_string_view haystack, basic_string_view needle ) -{ - return std::search( haystack.begin(), haystack.end(), needle.begin(), needle.end() ); -} - -#else // nssv_CONFIG_CONSTEXPR11_STD_SEARCH - -template< class CharT, class Traits = std::char_traits > -nssv_constexpr14 const CharT* search( basic_string_view haystack, basic_string_view needle ) -{ - while ( needle.size() <= haystack.size() ) - { - if ( haystack.starts_with(needle) ) - { - return haystack.cbegin(); - } - haystack = basic_string_view{ haystack.begin() + 1, haystack.size() - 1U }; - } - return haystack.cend(); -} -#endif // nssv_CONFIG_CONSTEXPR11_STD_SEARCH - -#endif // OPTIMIZE -#endif // nssv_CPP11_OR_GREATER && ! nssv_CPP17_OR_GREATER - -} // namespace detail - -// -// basic_string_view: -// - -template -< - class CharT, - class Traits /* = std::char_traits */ -> -class basic_string_view -{ -public: - // Member types: - - typedef Traits traits_type; - typedef CharT value_type; - - typedef CharT * pointer; - typedef CharT const * const_pointer; - typedef CharT & reference; - typedef CharT const & const_reference; - - typedef const_pointer iterator; - typedef const_pointer const_iterator; - typedef std::reverse_iterator< const_iterator > reverse_iterator; - typedef std::reverse_iterator< const_iterator > const_reverse_iterator; - - typedef std::size_t size_type; - typedef std::ptrdiff_t difference_type; - - // 24.4.2.1 Construction and assignment: - - nssv_constexpr basic_string_view() nssv_noexcept - : data_( nssv_nullptr ) - , size_( 0 ) - {} - -#if nssv_CPP11_OR_GREATER - nssv_constexpr basic_string_view( basic_string_view const & other ) nssv_noexcept = default; -#else - nssv_constexpr basic_string_view( basic_string_view const & other ) nssv_noexcept - : data_( other.data_) - , size_( other.size_) - {} -#endif - - nssv_constexpr basic_string_view( CharT const * s, size_type count ) nssv_noexcept // non-standard noexcept - : data_( s ) - , size_( count ) - {} - - nssv_constexpr basic_string_view( CharT const * s) nssv_noexcept // non-standard noexcept - : data_( s ) -#if nssv_CPP17_OR_GREATER - , size_( Traits::length(s) ) -#elif nssv_CPP11_OR_GREATER - , size_( detail::length(s) ) -#else - , size_( Traits::length(s) ) -#endif - {} - -#if nssv_HAVE_NULLPTR -# if nssv_HAVE_IS_DELETE - nssv_constexpr basic_string_view( std::nullptr_t ) nssv_noexcept = delete; -# else - private: nssv_constexpr basic_string_view( std::nullptr_t ) nssv_noexcept; public: -# endif -#endif - - // Assignment: - -#if nssv_CPP11_OR_GREATER - nssv_constexpr14 basic_string_view & operator=( basic_string_view const & other ) nssv_noexcept = default; -#else - nssv_constexpr14 basic_string_view & operator=( basic_string_view const & other ) nssv_noexcept - { - data_ = other.data_; - size_ = other.size_; - return *this; - } -#endif - - // 24.4.2.2 Iterator support: - - nssv_constexpr const_iterator begin() const nssv_noexcept { return data_; } - nssv_constexpr const_iterator end() const nssv_noexcept { return data_ + size_; } - - nssv_constexpr const_iterator cbegin() const nssv_noexcept { return begin(); } - nssv_constexpr const_iterator cend() const nssv_noexcept { return end(); } - - nssv_constexpr const_reverse_iterator rbegin() const nssv_noexcept { return const_reverse_iterator( end() ); } - nssv_constexpr const_reverse_iterator rend() const nssv_noexcept { return const_reverse_iterator( begin() ); } - - nssv_constexpr const_reverse_iterator crbegin() const nssv_noexcept { return rbegin(); } - nssv_constexpr const_reverse_iterator crend() const nssv_noexcept { return rend(); } - - // 24.4.2.3 Capacity: - - nssv_constexpr size_type size() const nssv_noexcept { return size_; } - nssv_constexpr size_type length() const nssv_noexcept { return size_; } - nssv_constexpr size_type max_size() const nssv_noexcept { return (std::numeric_limits< size_type >::max)(); } - - // since C++20 - nssv_nodiscard nssv_constexpr bool empty() const nssv_noexcept - { - return 0 == size_; - } - - // 24.4.2.4 Element access: - - nssv_constexpr const_reference operator[]( size_type pos ) const - { - return data_at( pos ); - } - - nssv_constexpr14 const_reference at( size_type pos ) const - { -#if nssv_CONFIG_NO_EXCEPTIONS - assert( pos < size() ); -#else - if ( pos >= size() ) - { - throw std::out_of_range("nonstd::string_view::at()"); - } -#endif - return data_at( pos ); - } - - nssv_constexpr const_reference front() const { return data_at( 0 ); } - nssv_constexpr const_reference back() const { return data_at( size() - 1 ); } - - nssv_constexpr const_pointer data() const nssv_noexcept { return data_; } - - // 24.4.2.5 Modifiers: - - nssv_constexpr14 void remove_prefix( size_type n ) - { - assert( n <= size() ); - data_ += n; - size_ -= n; - } - - nssv_constexpr14 void remove_suffix( size_type n ) - { - assert( n <= size() ); - size_ -= n; - } - - nssv_constexpr14 void swap( basic_string_view & other ) nssv_noexcept - { - const basic_string_view tmp(other); - other = *this; - *this = tmp; - } - - // 24.4.2.6 String operations: - - size_type copy( CharT * dest, size_type n, size_type pos = 0 ) const - { -#if nssv_CONFIG_NO_EXCEPTIONS - assert( pos <= size() ); -#else - if ( pos > size() ) - { - throw std::out_of_range("nonstd::string_view::copy()"); - } -#endif - const size_type rlen = (std::min)( n, size() - pos ); - - (void) Traits::copy( dest, data() + pos, rlen ); - - return rlen; - } - - nssv_constexpr14 basic_string_view substr( size_type pos = 0, size_type n = npos ) const - { -#if nssv_CONFIG_NO_EXCEPTIONS - assert( pos <= size() ); -#else - if ( pos > size() ) - { - throw std::out_of_range("nonstd::string_view::substr()"); - } -#endif - return basic_string_view( data() + pos, (std::min)( n, size() - pos ) ); - } - - // compare(), 6x: - - nssv_constexpr14 int compare( basic_string_view other ) const nssv_noexcept // (1) - { -#if nssv_CPP17_OR_GREATER - if ( const int result = Traits::compare( data(), other.data(), (std::min)( size(), other.size() ) ) ) -#else - if ( const int result = detail::compare( data(), other.data(), (std::min)( size(), other.size() ) ) ) -#endif - { - return result; - } - - return size() == other.size() ? 0 : size() < other.size() ? -1 : 1; - } - - nssv_constexpr int compare( size_type pos1, size_type n1, basic_string_view other ) const // (2) - { - return substr( pos1, n1 ).compare( other ); - } - - nssv_constexpr int compare( size_type pos1, size_type n1, basic_string_view other, size_type pos2, size_type n2 ) const // (3) - { - return substr( pos1, n1 ).compare( other.substr( pos2, n2 ) ); - } - - nssv_constexpr int compare( CharT const * s ) const // (4) - { - return compare( basic_string_view( s ) ); - } - - nssv_constexpr int compare( size_type pos1, size_type n1, CharT const * s ) const // (5) - { - return substr( pos1, n1 ).compare( basic_string_view( s ) ); - } - - nssv_constexpr int compare( size_type pos1, size_type n1, CharT const * s, size_type n2 ) const // (6) - { - return substr( pos1, n1 ).compare( basic_string_view( s, n2 ) ); - } - - // 24.4.2.7 Searching: - - // starts_with(), 3x, since C++20: - - nssv_constexpr bool starts_with( basic_string_view v ) const nssv_noexcept // (1) - { - return size() >= v.size() && compare( 0, v.size(), v ) == 0; - } - - nssv_constexpr bool starts_with( CharT c ) const nssv_noexcept // (2) - { - return starts_with( basic_string_view( &c, 1 ) ); - } - - nssv_constexpr bool starts_with( CharT const * s ) const // (3) - { - return starts_with( basic_string_view( s ) ); - } - - // ends_with(), 3x, since C++20: - - nssv_constexpr bool ends_with( basic_string_view v ) const nssv_noexcept // (1) - { - return size() >= v.size() && compare( size() - v.size(), npos, v ) == 0; - } - - nssv_constexpr bool ends_with( CharT c ) const nssv_noexcept // (2) - { - return ends_with( basic_string_view( &c, 1 ) ); - } - - nssv_constexpr bool ends_with( CharT const * s ) const // (3) - { - return ends_with( basic_string_view( s ) ); - } - - // find(), 4x: - - nssv_constexpr14 size_type find( basic_string_view v, size_type pos = 0 ) const nssv_noexcept // (1) - { - return assert( v.size() == 0 || v.data() != nssv_nullptr ) - , pos >= size() - ? npos : to_pos( -#if nssv_CPP11_OR_GREATER && ! nssv_CPP17_OR_GREATER - detail::search( substr(pos), v ) -#else - std::search( cbegin() + pos, cend(), v.cbegin(), v.cend(), Traits::eq ) -#endif - ); - } - - nssv_constexpr size_type find( CharT c, size_type pos = 0 ) const nssv_noexcept // (2) - { - return find( basic_string_view( &c, 1 ), pos ); - } - - nssv_constexpr size_type find( CharT const * s, size_type pos, size_type n ) const // (3) - { - return find( basic_string_view( s, n ), pos ); - } - - nssv_constexpr size_type find( CharT const * s, size_type pos = 0 ) const // (4) - { - return find( basic_string_view( s ), pos ); - } - - // rfind(), 4x: - - nssv_constexpr14 size_type rfind( basic_string_view v, size_type pos = npos ) const nssv_noexcept // (1) - { - if ( size() < v.size() ) - { - return npos; - } - - if ( v.empty() ) - { - return (std::min)( size(), pos ); - } - - const_iterator last = cbegin() + (std::min)( size() - v.size(), pos ) + v.size(); - const_iterator result = std::find_end( cbegin(), last, v.cbegin(), v.cend(), Traits::eq ); - - return result != last ? size_type( result - cbegin() ) : npos; - } - - nssv_constexpr14 size_type rfind( CharT c, size_type pos = npos ) const nssv_noexcept // (2) - { - return rfind( basic_string_view( &c, 1 ), pos ); - } - - nssv_constexpr14 size_type rfind( CharT const * s, size_type pos, size_type n ) const // (3) - { - return rfind( basic_string_view( s, n ), pos ); - } - - nssv_constexpr14 size_type rfind( CharT const * s, size_type pos = npos ) const // (4) - { - return rfind( basic_string_view( s ), pos ); - } - - // find_first_of(), 4x: - - nssv_constexpr size_type find_first_of( basic_string_view v, size_type pos = 0 ) const nssv_noexcept // (1) - { - return pos >= size() - ? npos - : to_pos( std::find_first_of( cbegin() + pos, cend(), v.cbegin(), v.cend(), Traits::eq ) ); - } - - nssv_constexpr size_type find_first_of( CharT c, size_type pos = 0 ) const nssv_noexcept // (2) - { - return find_first_of( basic_string_view( &c, 1 ), pos ); - } - - nssv_constexpr size_type find_first_of( CharT const * s, size_type pos, size_type n ) const // (3) - { - return find_first_of( basic_string_view( s, n ), pos ); - } - - nssv_constexpr size_type find_first_of( CharT const * s, size_type pos = 0 ) const // (4) - { - return find_first_of( basic_string_view( s ), pos ); - } - - // find_last_of(), 4x: - - nssv_constexpr size_type find_last_of( basic_string_view v, size_type pos = npos ) const nssv_noexcept // (1) - { - return empty() - ? npos - : pos >= size() - ? find_last_of( v, size() - 1 ) - : to_pos( std::find_first_of( const_reverse_iterator( cbegin() + pos + 1 ), crend(), v.cbegin(), v.cend(), Traits::eq ) ); - } - - nssv_constexpr size_type find_last_of( CharT c, size_type pos = npos ) const nssv_noexcept // (2) - { - return find_last_of( basic_string_view( &c, 1 ), pos ); - } - - nssv_constexpr size_type find_last_of( CharT const * s, size_type pos, size_type count ) const // (3) - { - return find_last_of( basic_string_view( s, count ), pos ); - } - - nssv_constexpr size_type find_last_of( CharT const * s, size_type pos = npos ) const // (4) - { - return find_last_of( basic_string_view( s ), pos ); - } - - // find_first_not_of(), 4x: - - nssv_constexpr size_type find_first_not_of( basic_string_view v, size_type pos = 0 ) const nssv_noexcept // (1) - { - return pos >= size() - ? npos - : to_pos( std::find_if( cbegin() + pos, cend(), not_in_view( v ) ) ); - } - - nssv_constexpr size_type find_first_not_of( CharT c, size_type pos = 0 ) const nssv_noexcept // (2) - { - return find_first_not_of( basic_string_view( &c, 1 ), pos ); - } - - nssv_constexpr size_type find_first_not_of( CharT const * s, size_type pos, size_type count ) const // (3) - { - return find_first_not_of( basic_string_view( s, count ), pos ); - } - - nssv_constexpr size_type find_first_not_of( CharT const * s, size_type pos = 0 ) const // (4) - { - return find_first_not_of( basic_string_view( s ), pos ); - } - - // find_last_not_of(), 4x: - - nssv_constexpr size_type find_last_not_of( basic_string_view v, size_type pos = npos ) const nssv_noexcept // (1) - { - return empty() - ? npos - : pos >= size() - ? find_last_not_of( v, size() - 1 ) - : to_pos( std::find_if( const_reverse_iterator( cbegin() + pos + 1 ), crend(), not_in_view( v ) ) ); - } - - nssv_constexpr size_type find_last_not_of( CharT c, size_type pos = npos ) const nssv_noexcept // (2) - { - return find_last_not_of( basic_string_view( &c, 1 ), pos ); - } - - nssv_constexpr size_type find_last_not_of( CharT const * s, size_type pos, size_type count ) const // (3) - { - return find_last_not_of( basic_string_view( s, count ), pos ); - } - - nssv_constexpr size_type find_last_not_of( CharT const * s, size_type pos = npos ) const // (4) - { - return find_last_not_of( basic_string_view( s ), pos ); - } - - // Constants: - -#if nssv_CPP17_OR_GREATER - static nssv_constexpr size_type npos = size_type(-1); -#elif nssv_CPP11_OR_GREATER - enum : size_type { npos = size_type(-1) }; -#else - enum { npos = size_type(-1) }; -#endif - -private: - struct not_in_view - { - const basic_string_view v; - - nssv_constexpr explicit not_in_view( basic_string_view v_ ) : v( v_ ) {} - - nssv_constexpr bool operator()( CharT c ) const - { - return npos == v.find_first_of( c ); - } - }; - - nssv_constexpr size_type to_pos( const_iterator it ) const - { - return it == cend() ? npos : size_type( it - cbegin() ); - } - - nssv_constexpr size_type to_pos( const_reverse_iterator it ) const - { - return it == crend() ? npos : size_type( crend() - it - 1 ); - } - - nssv_constexpr const_reference data_at( size_type pos ) const - { -#if nssv_BETWEEN( nssv_COMPILER_GNUC_VERSION, 1, 500 ) - return data_[pos]; -#else - return assert( pos < size() ), data_[pos]; -#endif - } - -private: - const_pointer data_; - size_type size_; - -public: -#if nssv_CONFIG_CONVERSION_STD_STRING_CLASS_METHODS - - template< class Allocator > - basic_string_view( std::basic_string const & s ) nssv_noexcept - : data_( s.data() ) - , size_( s.size() ) - {} - -#if nssv_HAVE_EXPLICIT_CONVERSION - - template< class Allocator > - explicit operator std::basic_string() const - { - return to_string( Allocator() ); - } - -#endif // nssv_HAVE_EXPLICIT_CONVERSION - -#if nssv_CPP11_OR_GREATER - - template< class Allocator = std::allocator > - std::basic_string - to_string( Allocator const & a = Allocator() ) const - { - return std::basic_string( begin(), end(), a ); - } - -#else - - std::basic_string - to_string() const - { - return std::basic_string( begin(), end() ); - } - - template< class Allocator > - std::basic_string - to_string( Allocator const & a ) const - { - return std::basic_string( begin(), end(), a ); - } - -#endif // nssv_CPP11_OR_GREATER - -#endif // nssv_CONFIG_CONVERSION_STD_STRING_CLASS_METHODS -}; - -// -// Non-member functions: -// - -// 24.4.3 Non-member comparison functions: -// lexicographically compare two string views (function template): - -template< class CharT, class Traits > -nssv_constexpr bool operator== ( - basic_string_view lhs, - basic_string_view rhs ) nssv_noexcept -{ return lhs.size() == rhs.size() && lhs.compare( rhs ) == 0; } - -template< class CharT, class Traits > -nssv_constexpr bool operator!= ( - basic_string_view lhs, - basic_string_view rhs ) nssv_noexcept -{ return !( lhs == rhs ); } - -template< class CharT, class Traits > -nssv_constexpr bool operator< ( - basic_string_view lhs, - basic_string_view rhs ) nssv_noexcept -{ return lhs.compare( rhs ) < 0; } - -template< class CharT, class Traits > -nssv_constexpr bool operator<= ( - basic_string_view lhs, - basic_string_view rhs ) nssv_noexcept -{ return lhs.compare( rhs ) <= 0; } - -template< class CharT, class Traits > -nssv_constexpr bool operator> ( - basic_string_view lhs, - basic_string_view rhs ) nssv_noexcept -{ return lhs.compare( rhs ) > 0; } - -template< class CharT, class Traits > -nssv_constexpr bool operator>= ( - basic_string_view lhs, - basic_string_view rhs ) nssv_noexcept -{ return lhs.compare( rhs ) >= 0; } - -// Let S be basic_string_view, and sv be an instance of S. -// Implementations shall provide sufficient additional overloads marked -// constexpr and noexcept so that an object t with an implicit conversion -// to S can be compared according to Table 67. - -#if ! nssv_CPP11_OR_GREATER || nssv_BETWEEN( nssv_COMPILER_MSVC_VERSION, 100, 141 ) - -// accommodate for older compilers: - -// == - -template< class CharT, class Traits> -nssv_constexpr bool operator==( - basic_string_view lhs, - CharT const * rhs ) nssv_noexcept -{ return lhs.size() == detail::length( rhs ) && lhs.compare( rhs ) == 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator==( - CharT const * lhs, - basic_string_view rhs ) nssv_noexcept -{ return detail::length( lhs ) == rhs.size() && rhs.compare( lhs ) == 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator==( - basic_string_view lhs, - std::basic_string rhs ) nssv_noexcept -{ return lhs.size() == rhs.size() && lhs.compare( rhs ) == 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator==( - std::basic_string rhs, - basic_string_view lhs ) nssv_noexcept -{ return lhs.size() == rhs.size() && lhs.compare( rhs ) == 0; } - -// != - -template< class CharT, class Traits> -nssv_constexpr bool operator!=( - basic_string_view lhs, - CharT const * rhs ) nssv_noexcept -{ return !( lhs == rhs ); } - -template< class CharT, class Traits> -nssv_constexpr bool operator!=( - CharT const * lhs, - basic_string_view rhs ) nssv_noexcept -{ return !( lhs == rhs ); } - -template< class CharT, class Traits> -nssv_constexpr bool operator!=( - basic_string_view lhs, - std::basic_string rhs ) nssv_noexcept -{ return !( lhs == rhs ); } - -template< class CharT, class Traits> -nssv_constexpr bool operator!=( - std::basic_string rhs, - basic_string_view lhs ) nssv_noexcept -{ return !( lhs == rhs ); } - -// < - -template< class CharT, class Traits> -nssv_constexpr bool operator<( - basic_string_view lhs, - CharT const * rhs ) nssv_noexcept -{ return lhs.compare( rhs ) < 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator<( - CharT const * lhs, - basic_string_view rhs ) nssv_noexcept -{ return rhs.compare( lhs ) > 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator<( - basic_string_view lhs, - std::basic_string rhs ) nssv_noexcept -{ return lhs.compare( rhs ) < 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator<( - std::basic_string rhs, - basic_string_view lhs ) nssv_noexcept -{ return rhs.compare( lhs ) > 0; } - -// <= - -template< class CharT, class Traits> -nssv_constexpr bool operator<=( - basic_string_view lhs, - CharT const * rhs ) nssv_noexcept -{ return lhs.compare( rhs ) <= 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator<=( - CharT const * lhs, - basic_string_view rhs ) nssv_noexcept -{ return rhs.compare( lhs ) >= 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator<=( - basic_string_view lhs, - std::basic_string rhs ) nssv_noexcept -{ return lhs.compare( rhs ) <= 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator<=( - std::basic_string rhs, - basic_string_view lhs ) nssv_noexcept -{ return rhs.compare( lhs ) >= 0; } - -// > - -template< class CharT, class Traits> -nssv_constexpr bool operator>( - basic_string_view lhs, - CharT const * rhs ) nssv_noexcept -{ return lhs.compare( rhs ) > 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator>( - CharT const * lhs, - basic_string_view rhs ) nssv_noexcept -{ return rhs.compare( lhs ) < 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator>( - basic_string_view lhs, - std::basic_string rhs ) nssv_noexcept -{ return lhs.compare( rhs ) > 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator>( - std::basic_string rhs, - basic_string_view lhs ) nssv_noexcept -{ return rhs.compare( lhs ) < 0; } - -// >= - -template< class CharT, class Traits> -nssv_constexpr bool operator>=( - basic_string_view lhs, - CharT const * rhs ) nssv_noexcept -{ return lhs.compare( rhs ) >= 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator>=( - CharT const * lhs, - basic_string_view rhs ) nssv_noexcept -{ return rhs.compare( lhs ) <= 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator>=( - basic_string_view lhs, - std::basic_string rhs ) nssv_noexcept -{ return lhs.compare( rhs ) >= 0; } - -template< class CharT, class Traits> -nssv_constexpr bool operator>=( - std::basic_string rhs, - basic_string_view lhs ) nssv_noexcept -{ return rhs.compare( lhs ) <= 0; } - -#else // newer compilers: - -#define nssv_BASIC_STRING_VIEW_I(T,U) typename std::decay< basic_string_view >::type - -#if defined(_MSC_VER) // issue 40 -# define nssv_MSVC_ORDER(x) , int=x -#else -# define nssv_MSVC_ORDER(x) /*, int=x*/ -#endif - -// == - -template< class CharT, class Traits nssv_MSVC_ORDER(1) > -nssv_constexpr bool operator==( - basic_string_view lhs, - nssv_BASIC_STRING_VIEW_I(CharT, Traits) rhs ) nssv_noexcept -{ return lhs.size() == rhs.size() && lhs.compare( rhs ) == 0; } - -template< class CharT, class Traits nssv_MSVC_ORDER(2) > -nssv_constexpr bool operator==( - nssv_BASIC_STRING_VIEW_I(CharT, Traits) lhs, - basic_string_view rhs ) nssv_noexcept -{ return lhs.size() == rhs.size() && lhs.compare( rhs ) == 0; } - -// != - -template< class CharT, class Traits nssv_MSVC_ORDER(1) > -nssv_constexpr bool operator!= ( - basic_string_view < CharT, Traits > lhs, - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) rhs ) nssv_noexcept -{ return !( lhs == rhs ); } - -template< class CharT, class Traits nssv_MSVC_ORDER(2) > -nssv_constexpr bool operator!= ( - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) lhs, - basic_string_view < CharT, Traits > rhs ) nssv_noexcept -{ return !( lhs == rhs ); } - -// < - -template< class CharT, class Traits nssv_MSVC_ORDER(1) > -nssv_constexpr bool operator< ( - basic_string_view < CharT, Traits > lhs, - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) rhs ) nssv_noexcept -{ return lhs.compare( rhs ) < 0; } - -template< class CharT, class Traits nssv_MSVC_ORDER(2) > -nssv_constexpr bool operator< ( - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) lhs, - basic_string_view < CharT, Traits > rhs ) nssv_noexcept -{ return lhs.compare( rhs ) < 0; } - -// <= - -template< class CharT, class Traits nssv_MSVC_ORDER(1) > -nssv_constexpr bool operator<= ( - basic_string_view < CharT, Traits > lhs, - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) rhs ) nssv_noexcept -{ return lhs.compare( rhs ) <= 0; } - -template< class CharT, class Traits nssv_MSVC_ORDER(2) > -nssv_constexpr bool operator<= ( - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) lhs, - basic_string_view < CharT, Traits > rhs ) nssv_noexcept -{ return lhs.compare( rhs ) <= 0; } - -// > - -template< class CharT, class Traits nssv_MSVC_ORDER(1) > -nssv_constexpr bool operator> ( - basic_string_view < CharT, Traits > lhs, - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) rhs ) nssv_noexcept -{ return lhs.compare( rhs ) > 0; } - -template< class CharT, class Traits nssv_MSVC_ORDER(2) > -nssv_constexpr bool operator> ( - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) lhs, - basic_string_view < CharT, Traits > rhs ) nssv_noexcept -{ return lhs.compare( rhs ) > 0; } - -// >= - -template< class CharT, class Traits nssv_MSVC_ORDER(1) > -nssv_constexpr bool operator>= ( - basic_string_view < CharT, Traits > lhs, - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) rhs ) nssv_noexcept -{ return lhs.compare( rhs ) >= 0; } - -template< class CharT, class Traits nssv_MSVC_ORDER(2) > -nssv_constexpr bool operator>= ( - nssv_BASIC_STRING_VIEW_I( CharT, Traits ) lhs, - basic_string_view < CharT, Traits > rhs ) nssv_noexcept -{ return lhs.compare( rhs ) >= 0; } - -#undef nssv_MSVC_ORDER -#undef nssv_BASIC_STRING_VIEW_I - -#endif // compiler-dependent approach to comparisons - -// 24.4.4 Inserters and extractors: - -#if ! nssv_CONFIG_NO_STREAM_INSERTION - -namespace detail { - -template< class Stream > -void write_padding( Stream & os, std::streamsize n ) -{ - for ( std::streamsize i = 0; i < n; ++i ) - os.rdbuf()->sputc( os.fill() ); -} - -template< class Stream, class View > -Stream & write_to_stream( Stream & os, View const & sv ) -{ - typename Stream::sentry sentry( os ); - - if ( !sentry ) - return os; - - const std::streamsize length = static_cast( sv.length() ); - - // Whether, and how, to pad: - const bool pad = ( length < os.width() ); - const bool left_pad = pad && ( os.flags() & std::ios_base::adjustfield ) == std::ios_base::right; - - if ( left_pad ) - write_padding( os, os.width() - length ); - - // Write span characters: - os.rdbuf()->sputn( sv.begin(), length ); - - if ( pad && !left_pad ) - write_padding( os, os.width() - length ); - - // Reset output stream width: - os.width( 0 ); - - return os; -} - -} // namespace detail - -template< class CharT, class Traits > -std::basic_ostream & -operator<<( - std::basic_ostream& os, - basic_string_view sv ) -{ - return detail::write_to_stream( os, sv ); -} - -#endif // nssv_CONFIG_NO_STREAM_INSERTION - -// Several typedefs for common character types are provided: - -typedef basic_string_view string_view; -typedef basic_string_view wstring_view; -#if nssv_HAVE_WCHAR16_T -typedef basic_string_view u16string_view; -typedef basic_string_view u32string_view; -#endif - -}} // namespace nonstd::sv_lite - -// -// 24.4.6 Suffix for basic_string_view literals: -// - -#if nssv_HAVE_USER_DEFINED_LITERALS - -namespace nonstd { -nssv_inline_ns namespace literals { -nssv_inline_ns namespace string_view_literals { - -#if nssv_CONFIG_STD_SV_OPERATOR && nssv_HAVE_STD_DEFINED_LITERALS - -nssv_constexpr nonstd::sv_lite::string_view operator ""sv( const char* str, size_t len ) nssv_noexcept // (1) -{ - return nonstd::sv_lite::string_view{ str, len }; -} - -nssv_constexpr nonstd::sv_lite::u16string_view operator ""sv( const char16_t* str, size_t len ) nssv_noexcept // (2) -{ - return nonstd::sv_lite::u16string_view{ str, len }; -} - -nssv_constexpr nonstd::sv_lite::u32string_view operator ""sv( const char32_t* str, size_t len ) nssv_noexcept // (3) -{ - return nonstd::sv_lite::u32string_view{ str, len }; -} - -nssv_constexpr nonstd::sv_lite::wstring_view operator ""sv( const wchar_t* str, size_t len ) nssv_noexcept // (4) -{ - return nonstd::sv_lite::wstring_view{ str, len }; -} - -#endif // nssv_CONFIG_STD_SV_OPERATOR && nssv_HAVE_STD_DEFINED_LITERALS - -#if nssv_CONFIG_USR_SV_OPERATOR - -nssv_constexpr nonstd::sv_lite::string_view operator ""_sv( const char* str, size_t len ) nssv_noexcept // (1) -{ - return nonstd::sv_lite::string_view{ str, len }; -} - -nssv_constexpr nonstd::sv_lite::u16string_view operator ""_sv( const char16_t* str, size_t len ) nssv_noexcept // (2) -{ - return nonstd::sv_lite::u16string_view{ str, len }; -} - -nssv_constexpr nonstd::sv_lite::u32string_view operator ""_sv( const char32_t* str, size_t len ) nssv_noexcept // (3) -{ - return nonstd::sv_lite::u32string_view{ str, len }; -} - -nssv_constexpr nonstd::sv_lite::wstring_view operator ""_sv( const wchar_t* str, size_t len ) nssv_noexcept // (4) -{ - return nonstd::sv_lite::wstring_view{ str, len }; -} - -#endif // nssv_CONFIG_USR_SV_OPERATOR - -}}} // namespace nonstd::literals::string_view_literals - -#endif - -// -// Extensions for std::string: -// - -#if nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS - -namespace nonstd { -namespace sv_lite { - -// Exclude MSVC 14 (19.00): it yields ambiguous to_string(): - -#if nssv_CPP11_OR_GREATER && nssv_COMPILER_MSVC_VERSION != 140 - -template< class CharT, class Traits, class Allocator = std::allocator > -std::basic_string -to_string( basic_string_view v, Allocator const & a = Allocator() ) -{ - return std::basic_string( v.begin(), v.end(), a ); -} - -#else - -template< class CharT, class Traits > -std::basic_string -to_string( basic_string_view v ) -{ - return std::basic_string( v.begin(), v.end() ); -} - -template< class CharT, class Traits, class Allocator > -std::basic_string -to_string( basic_string_view v, Allocator const & a ) -{ - return std::basic_string( v.begin(), v.end(), a ); -} - -#endif // nssv_CPP11_OR_GREATER - -template< class CharT, class Traits, class Allocator > -basic_string_view -to_string_view( std::basic_string const & s ) -{ - return basic_string_view( s.data(), s.size() ); -} - -}} // namespace nonstd::sv_lite - -#endif // nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS - -// -// make types and algorithms available in namespace nonstd: -// - -namespace nonstd { - -using sv_lite::basic_string_view; -using sv_lite::string_view; -using sv_lite::wstring_view; - -#if nssv_HAVE_WCHAR16_T -using sv_lite::u16string_view; -#endif -#if nssv_HAVE_WCHAR32_T -using sv_lite::u32string_view; -#endif - -// literal "sv" - -using sv_lite::operator==; -using sv_lite::operator!=; -using sv_lite::operator<; -using sv_lite::operator<=; -using sv_lite::operator>; -using sv_lite::operator>=; - -#if ! nssv_CONFIG_NO_STREAM_INSERTION -using sv_lite::operator<<; -#endif - -#if nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS -using sv_lite::to_string; -using sv_lite::to_string_view; -#endif - -} // namespace nonstd - -// 24.4.5 Hash support (C++11): - -// Note: The hash value of a string view object is equal to the hash value of -// the corresponding string object. - -#if nssv_HAVE_STD_HASH - -#include - -namespace std { - -template<> -struct hash< nonstd::string_view > -{ -public: - std::size_t operator()( nonstd::string_view v ) const nssv_noexcept - { - return std::hash()( std::string( v.data(), v.size() ) ); - } -}; - -template<> -struct hash< nonstd::wstring_view > -{ -public: - std::size_t operator()( nonstd::wstring_view v ) const nssv_noexcept - { - return std::hash()( std::wstring( v.data(), v.size() ) ); - } -}; - -template<> -struct hash< nonstd::u16string_view > -{ -public: - std::size_t operator()( nonstd::u16string_view v ) const nssv_noexcept - { - return std::hash()( std::u16string( v.data(), v.size() ) ); - } -}; - -template<> -struct hash< nonstd::u32string_view > -{ -public: - std::size_t operator()( nonstd::u32string_view v ) const nssv_noexcept - { - return std::hash()( std::u32string( v.data(), v.size() ) ); - } -}; - -} // namespace std - -#endif // nssv_HAVE_STD_HASH - -nssv_RESTORE_WARNINGS() - -#endif // nssv_HAVE_STD_STRING_VIEW -#endif // NONSTD_SV_LITE_H_INCLUDED -/* end file simdjson/nonstd/string_view.hpp */ -SIMDJSON_POP_DISABLE_WARNINGS - -namespace std { - using string_view = nonstd::string_view; -} -#endif // SIMDJSON_HAS_STRING_VIEW -#undef SIMDJSON_HAS_STRING_VIEW // We are not going to need this macro anymore. - -/// If EXPR is an error, returns it. -#define SIMDJSON_TRY(EXPR) { auto _err = (EXPR); if (_err) { return _err; } } - -// Unless the programmer has already set SIMDJSON_DEVELOPMENT_CHECKS, -// we want to set it under debug builds. We detect a debug build -// under Visual Studio when the _DEBUG macro is set. Under the other -// compilers, we use the fact that they define __OPTIMIZE__ whenever -// they allow optimizations. -// It is possible that this could miss some cases where SIMDJSON_DEVELOPMENT_CHECKS -// is helpful, but the programmer can set the macro SIMDJSON_DEVELOPMENT_CHECKS. -// It could also wrongly set SIMDJSON_DEVELOPMENT_CHECKS (e.g., if the programmer -// sets _DEBUG in a release build under Visual Studio, or if some compiler fails to -// set the __OPTIMIZE__ macro). -// We make it so that if NDEBUG is defined, then SIMDJSON_DEVELOPMENT_CHECKS -// is not defined, irrespective of the compiler. -// We recommend that users set NDEBUG in release builds, so that -// SIMDJSON_DEVELOPMENT_CHECKS is not defined in release builds by default, -// irrespective of the compiler. -#ifndef SIMDJSON_DEVELOPMENT_CHECKS -#ifdef _MSC_VER -// Visual Studio seems to set _DEBUG for debug builds. -// We set SIMDJSON_DEVELOPMENT_CHECKS to 1 if _DEBUG is defined -// and NDEBUG is not defined. -#if defined(_DEBUG) && !defined(NDEBUG) -#define SIMDJSON_DEVELOPMENT_CHECKS 1 -#endif // _DEBUG -#else // _MSC_VER -// All other compilers appear to set __OPTIMIZE__ to a positive integer -// when the compiler is optimizing. -// We only set SIMDJSON_DEVELOPMENT_CHECKS if both __OPTIMIZE__ -// and NDEBUG are not defined. -// We recognize _DEBUG as overriding __OPTIMIZE__ so that if both -// __OPTIMIZE__ and _DEBUG are defined, we still set SIMDJSON_DEVELOPMENT_CHECKS. -#if ((!defined(__OPTIMIZE__) || defined(_DEBUG)) && !defined(NDEBUG)) -#define SIMDJSON_DEVELOPMENT_CHECKS 1 -#endif // __OPTIMIZE__ -#endif // _MSC_VER -#endif // SIMDJSON_DEVELOPMENT_CHECKS - -// The SIMDJSON_CHECK_EOF macro is a feature flag for the "don't require padding" -// feature. - -#if SIMDJSON_CPLUSPLUS17 -// if we have C++, then fallthrough is a default attribute -# define simdjson_fallthrough [[fallthrough]] -// check if we have __attribute__ support -#elif defined(__has_attribute) -// check if we have the __fallthrough__ attribute -#if __has_attribute(__fallthrough__) -// we are good to go: -# define simdjson_fallthrough __attribute__((__fallthrough__)) -#endif // __has_attribute(__fallthrough__) -#endif // SIMDJSON_CPLUSPLUS17 -// on some systems, we simply do not have support for fallthrough, so use a default: -#ifndef simdjson_fallthrough -# define simdjson_fallthrough do {} while (0) /* fallthrough */ -#endif // simdjson_fallthrough - -#if SIMDJSON_DEVELOPMENT_CHECKS -#define SIMDJSON_DEVELOPMENT_ASSERT(expr) do { assert ((expr)); } while (0) -#else -#define SIMDJSON_DEVELOPMENT_ASSERT(expr) do { } while (0) -#endif - -#ifndef SIMDJSON_UTF8VALIDATION -#define SIMDJSON_UTF8VALIDATION 1 -#endif - -#ifdef __has_include -// How do we detect that a compiler supports vbmi2? -// For sure if the following header is found, we are ok? -#if __has_include() -#define SIMDJSON_COMPILER_SUPPORTS_VBMI2 1 -#endif -#endif - -#ifdef _MSC_VER -#if _MSC_VER >= 1920 -// Visual Studio 2019 and up support VBMI2 under x64 even if the header -// avx512vbmi2intrin.h is not found. -#define SIMDJSON_COMPILER_SUPPORTS_VBMI2 1 -#endif -#endif - -// By default, we allow AVX512. -#ifndef SIMDJSON_AVX512_ALLOWED -#define SIMDJSON_AVX512_ALLOWED 1 -#endif - - -#ifndef __has_cpp_attribute -#define simdjson_lifetime_bound -#elif __has_cpp_attribute(msvc::lifetimebound) -#define simdjson_lifetime_bound [[msvc::lifetimebound]] -#elif __has_cpp_attribute(clang::lifetimebound) -#define simdjson_lifetime_bound [[clang::lifetimebound]] -#elif __has_cpp_attribute(lifetimebound) -#define simdjson_lifetime_bound [[lifetimebound]] -#else -#define simdjson_lifetime_bound -#endif -#endif // SIMDJSON_COMMON_DEFS_H -/* end file simdjson/common_defs.h */ - -// This provides the public API for simdjson. -// DOM and ondemand are amalgamated separately, in simdjson.h -/* including simdjson/simdjson_version.h: #include "simdjson/simdjson_version.h" */ -/* begin file simdjson/simdjson_version.h */ -// /include/simdjson/simdjson_version.h automatically generated by release.py, -// do not change by hand -#ifndef SIMDJSON_SIMDJSON_VERSION_H -#define SIMDJSON_SIMDJSON_VERSION_H - -/** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION "5.1.1" - -namespace simdjson { -enum { - /** - * The major version (MAJOR.minor.revision) of simdjson being used. - */ - SIMDJSON_VERSION_MAJOR = 5, - /** - * The minor version (major.MINOR.revision) of simdjson being used. - */ - SIMDJSON_VERSION_MINOR = 1, - /** - * The revision (major.minor.REVISION) of simdjson being used. - */ - SIMDJSON_VERSION_REVISION = 1 -}; -} // namespace simdjson - -#endif // SIMDJSON_SIMDJSON_VERSION_H -/* end file simdjson/simdjson_version.h */ - -/* including simdjson/base.h: #include "simdjson/base.h" */ -/* begin file simdjson/base.h */ -/** - * @file Base declarations for all simdjson headers - * @private - */ -#ifndef SIMDJSON_BASE_H -#define SIMDJSON_BASE_H - -/* skipped duplicate #include "simdjson/common_defs.h" */ -/* skipped duplicate #include "simdjson/compiler_check.h" */ -/* including simdjson/error.h: #include "simdjson/error.h" */ -/* begin file simdjson/error.h */ -#ifndef SIMDJSON_ERROR_H -#define SIMDJSON_ERROR_H - -/* skipped duplicate #include "simdjson/base.h" */ - -#include -#include - -namespace simdjson { - -/** - * All possible errors returned by simdjson. These error codes are subject to change - * and not all simdjson kernel returns the same error code given the same input: it is not - * well defined which error a given input should produce. - * - * Only SUCCESS evaluates to false as a Boolean. All other error codes will evaluate - * to true as a Boolean. - */ -enum error_code { - SUCCESS = 0, ///< No error - CAPACITY, ///< This parser can't support a document that big - MEMALLOC, ///< Error allocating memory, most likely out of memory - TAPE_ERROR, ///< Something went wrong, this is a generic error. Fatal/unrecoverable error. - DEPTH_ERROR, ///< Your document exceeds the user-specified depth limitation - STRING_ERROR, ///< Problem while parsing a string - T_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 't' - F_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'f' - N_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'n' - NUMBER_ERROR, ///< Problem while parsing a number - BIGINT_ERROR, ///< The integer value exceeds 64 bits - UTF8_ERROR, ///< the input is not valid UTF-8 - UNINITIALIZED, ///< unknown error, or uninitialized document - EMPTY, ///< no structural element found - UNESCAPED_CHARS, ///< found unescaped characters in a string. - UNCLOSED_STRING, ///< missing quote at the end - UNSUPPORTED_ARCHITECTURE, ///< unsupported architecture - INCORRECT_TYPE, ///< JSON element has a different type than user expected - NUMBER_OUT_OF_RANGE, ///< JSON number does not fit in 64 bits - INDEX_OUT_OF_BOUNDS, ///< JSON array index too large - NO_SUCH_FIELD, ///< JSON field not found in object - IO_ERROR, ///< Error reading a file - INVALID_JSON_POINTER, ///< Invalid JSON pointer syntax - INVALID_URI_FRAGMENT, ///< Invalid URI fragment - UNEXPECTED_ERROR, ///< indicative of a bug in simdjson - PARSER_IN_USE, ///< parser is already in use. - OUT_OF_ORDER_ITERATION, ///< tried to iterate an array or object out of order (checked when SIMDJSON_DEVELOPMENT_CHECKS=1) - INSUFFICIENT_PADDING, ///< The JSON doesn't have enough padding for simdjson to safely parse it. - INCOMPLETE_ARRAY_OR_OBJECT, ///< The document ends early. Fatal/unrecoverable error. - SCALAR_DOCUMENT_AS_VALUE, ///< A scalar document is treated as a value. - OUT_OF_BOUNDS, ///< Attempted to access location outside of document. - TRAILING_CONTENT, ///< Unexpected trailing content in the JSON input - OUT_OF_CAPACITY, ///< The capacity was exceeded, we cannot allocate enough memory. - UNKNOWN_FIELD, ///< JSON field does not map to any member of the target (see simdjson::deny_unknown_fields) - NUM_ERROR_CODES ///< Placeholder for end of error code list. -}; - -/** - * Some errors are fatal and invalidate the document. This function returns true if the - * error is fatal. It returns true for TAPE_ERROR and INCOMPLETE_ARRAY_OR_OBJECT. - * Once a fatal error is encountered, the on-demand document is no longer valid and - * processing should stop. - */ - inline bool is_fatal(error_code error) noexcept; - -/** - * It is the convention throughout the code that the macro SIMDJSON_DEVELOPMENT_CHECKS determines whether - * we check for OUT_OF_ORDER_ITERATION. The logic behind it is that these errors only occurs when the code - * that was written while breaking some simdjson::ondemand requirement. They should not occur in released - * code after these issues were fixed. - */ - -/** - * Get the error message for the given error code. - * - * dom::parser parser; - * dom::element doc; - * auto error = parser.parse("foo",3).get(doc); - * if (error) { printf("Error: %s\n", error_message(error)); } - * - * @return The error message. - */ -inline const char *error_message(error_code error) noexcept; - -/** - * Write the error message to the output stream - */ -inline std::ostream& operator<<(std::ostream& out, error_code error) noexcept; - -/** - * Exception thrown when an exception-supporting simdjson method is called - */ -struct simdjson_error : public std::exception { - /** - * Create an exception from a simdjson error code. - * @param error The error code - */ - simdjson_error(error_code error) noexcept : _error{error} { } - /** The error message */ - const char *what() const noexcept override { return error_message(error()); } - /** The error code */ - error_code error() const noexcept { return _error; } -private: - /** The error code that was used */ - error_code _error; -}; - -namespace internal { - -/** - * The result of a simdjson operation that could fail. - * - * IMPORTANT: For the ondemand API, we use implementation_simdjson_result_base as a base class - * to avoid some compilation issue. Thus, if you modify this class, please ensure that the ondemand - * implementation_simdjson_result_base is also modified. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::simdjson_result_base { - * simdjson_result() noexcept : internal::simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct simdjson_result_base : protected std::pair { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline simdjson_result_base() noexcept; - - /** - * Create a new error result. - */ - simdjson_inline simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_inline bool has_value() const noexcept; -#if SIMDJSON_EXCEPTIONS - - /** - * Dereference operator to access the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - * We discourage the use of value_unsafe(). - * - * The recommended pattern is: - * - * T value; // where T is the type - * auto error = result.get(value); - * if (error) { - * // handle error - * } - * - * Or you may call 'value()' which will raise an exception - * in case of error: - * - * T value = result.value(); - */ - simdjson_inline const T& value_unsafe() const& noexcept; - - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - * We discourage the use of value_unsafe(). - * - * The recommended pattern is: - * - * T value; // where T is the type - * auto error = result.get(value); - * if (error) { - * // handle error, return, exit, abort - * } else { - * // use value here. - * } - * - * Or you may call 'value()' which will raise an exception - * in case of error: - * - * T value = result.value(); - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; -}; // struct simdjson_result_base - -} // namespace internal - -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - */ -template -struct simdjson_result : public internal::simdjson_result_base { - - /** - * @private Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline simdjson_result() noexcept; - /** - * @private Create a new successful result. - */ - simdjson_inline simdjson_result(T &&value) noexcept; - /** - * @private Create a new error result. - */ - simdjson_inline simdjson_result(error_code error_code) noexcept; - /** - * @private Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline simdjson_result(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * Copy the value to a provided std::string, only enabled for std::string_view. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - template - simdjson_warn_unused simdjson_inline error_code get(std::string &value) && noexcept { - static_assert(std::is_same::value, "SFINAE"); - std::string_view v; - error_code error = std::forward>(*this).get(v); - if (!error) { - value.assign(v.data(), v.size()); - } - return error; - } - - /** - * The error. - */ - simdjson_inline error_code error() const noexcept; - - - -#if SIMDJSON_EXCEPTIONS - using internal::simdjson_result_base::operator*; - using internal::simdjson_result_base::operator->; - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; -}; // struct simdjson_result - -#if SIMDJSON_EXCEPTIONS - -template -inline std::ostream& operator<<(std::ostream& out, simdjson_result value) { return out << value.value(); } -#endif // SIMDJSON_EXCEPTIONS - -#ifndef SIMDJSON_DISABLE_DEPRECATED_API -/** - * @deprecated This is an alias and will be removed, use error_code instead - */ -using ErrorValues [[deprecated("This is an alias and will be removed, use error_code instead")]] = error_code; - -/** - * @deprecated Error codes should be stored and returned as `error_code`, use `error_message()` instead. - */ -[[deprecated("Error codes should be stored and returned as `error_code`, use `error_message()` instead.")]] -inline const std::string error_message(int error) noexcept; -#endif // SIMDJSON_DISABLE_DEPRECATED_API -} // namespace simdjson - -#endif // SIMDJSON_ERROR_H -/* end file simdjson/error.h */ -/* skipped duplicate #include "simdjson/portability.h" */ -/* including simdjson/concepts.h: #include "simdjson/concepts.h" */ -/* begin file simdjson/concepts.h */ -#ifndef SIMDJSON_CONCEPTS_H -#define SIMDJSON_CONCEPTS_H -#if SIMDJSON_SUPPORTS_CONCEPTS - -#include -#include -#include - -namespace simdjson { -namespace concepts { - -namespace details { -#define SIMDJSON_IMPL_CONCEPT(name, method) \ - template \ - concept supports_##name = !std::is_const_v && requires { \ - typename std::remove_cvref_t::value_type; \ - requires requires(typename std::remove_cvref_t::value_type &&val, \ - T obj) { \ - obj.method(std::move(val)); \ - requires !requires { obj = std::move(val); }; \ - }; \ - }; - -SIMDJSON_IMPL_CONCEPT(emplace_back, emplace_back) -SIMDJSON_IMPL_CONCEPT(emplace, emplace) -SIMDJSON_IMPL_CONCEPT(push_back, push_back) -SIMDJSON_IMPL_CONCEPT(add, add) -SIMDJSON_IMPL_CONCEPT(push, push) -SIMDJSON_IMPL_CONCEPT(append, append) -SIMDJSON_IMPL_CONCEPT(insert, insert) -SIMDJSON_IMPL_CONCEPT(op_append, operator+=) - -#undef SIMDJSON_IMPL_CONCEPT -} // namespace details - -template -concept is_pair = requires { typename T::first_type; typename T::second_type; } && - std::same_as>; -template -concept string_view_like = std::is_convertible_v && - !std::is_convertible_v; - -template -concept constructible_from_string_view = std::is_constructible_v - && !std::is_same_v - && std::is_default_constructible_v; - -#if SIMDJSON_SUPPORTS_CHAR8_T -/** - * A C++20 char8_t string type such as std::u8string. Such types cannot be built - * from a std::string_view (the character types differ), so they need their own - * deserialization path, going through the u8 string accessors. - */ -template -concept constructible_from_u8string_view = std::is_constructible_v - && !std::is_same_v - && !std::is_constructible_v - && std::is_default_constructible_v; -#endif // SIMDJSON_SUPPORTS_CHAR8_T - -template -concept string_view_keyed_map = string_view_like - && requires(std::remove_cvref_t& m, typename M::key_type sv, typename M::mapped_type v) { - { m.emplace(sv, v) } -> std::same_as>; -}; - -/// Check if T is a container that we can append to, including: -/// std::vector, std::deque, std::list, std::string, ... -template -concept appendable_containers = - (details::supports_emplace_back || details::supports_emplace || - details::supports_push_back || details::supports_push || - details::supports_add || details::supports_append || - details::supports_insert) && !string_view_keyed_map; - -/// Insert into the container however possible -template -constexpr decltype(auto) emplace_one(T &vec, Args &&...args) { - if constexpr (details::supports_emplace_back) { - return vec.emplace_back(std::forward(args)...); - } else if constexpr (details::supports_emplace) { - return vec.emplace(std::forward(args)...); - } else if constexpr (details::supports_push_back) { - return vec.push_back(std::forward(args)...); - } else if constexpr (details::supports_push) { - return vec.push(std::forward(args)...); - } else if constexpr (details::supports_add) { - return vec.add(std::forward(args)...); - } else if constexpr (details::supports_append) { - return vec.append(std::forward(args)...); - } else if constexpr (details::supports_insert) { - return vec.insert(std::forward(args)...); - } else if constexpr (details::supports_op_append && sizeof...(Args) == 1) { - return vec.operator+=(std::forward(args)...); - } else { - static_assert(!sizeof(T *), - "We don't know how to add things to this container"); - } -} - -/// This checks if the container will return a reference to the newly added -/// element after an insert which for example `std::vector::emplace_back` does -/// since C++17; this will allow some optimizations. -template -concept returns_reference = appendable_containers && requires { - typename std::remove_cvref_t::reference; - requires requires(typename std::remove_cvref_t::value_type &&val, T obj) { - { - emplace_one(obj, std::move(val)) - } -> std::same_as::reference>; - }; -}; - -template -concept smart_pointer = requires(std::remove_cvref_t ptr) { - // Check if T has a member type named element_type - typename std::remove_cvref_t::element_type; - - // Check if T has a get() member function - { - ptr.get() - } -> std::same_as::element_type *>; - - // Check if T can be dereferenced - { *ptr } -> std::same_as::element_type &>; -}; - -template -concept optional_type = requires(std::remove_cvref_t obj) { - typename std::remove_cvref_t::value_type; - { obj.value() } -> std::same_as::value_type&>; - requires requires(typename std::remove_cvref_t::value_type &&val) { - obj.emplace(std::move(val)); - { - obj.value_or(val) - } -> std::convertible_to::value_type>; - }; - { static_cast(obj) } -> std::same_as; // convertible to bool - { obj.reset() } noexcept -> std::same_as; -}; - - - -// Types we serialize as JSON strings (not as containers) -template -concept string_like = - std::is_same_v, std::string> || - std::is_same_v, std::string_view> || - std::is_same_v, const char*> || - std::is_same_v, char*>; - -// Concept that checks if a type is a container but not a string (because -// strings handling must be handled differently) -// Now uses iterator-based approach for broader container support -// -// Optional types are excluded on purpose. Since C++26 (P3168), std::optional -// is itself a range, so without the exclusion an std::optional would match -// both this concept and optional_type, making the container and the optional -// overloads of atom()/append() ambiguous. See issue 2827. -template -concept container_but_not_string = - std::ranges::input_range && !string_like && !concepts::string_view_keyed_map - && !concepts::optional_type; - - - -// Concept: Indexable container that is not a string or associative container -// Accepts: std::vector, std::array, std::deque (have operator[], value_type, not string_like) -// Rejects: std::string (string_like), std::list (no operator[]), std::map (has key_type) -template -concept indexable_container = requires { - typename Container::value_type; - requires !concepts::string_like; - requires !requires { typename Container::key_type; }; // Reject maps/sets - requires requires(Container& c, std::size_t i) { - { c[i] } -> std::convertible_to; - }; -}; - - -// Variable template to use with std::meta::substitute -template -constexpr bool indexable_container_v = indexable_container; - -} // namespace concepts - - -/** - * We use tag_invoke as our customization point mechanism. - */ -template -concept tag_invocable = requires(Tag tag, Args... args) { - tag_invoke(std::forward(tag), std::forward(args)...); -}; - -template -concept nothrow_tag_invocable = - tag_invocable && requires(Tag tag, Args... args) { - { - tag_invoke(std::forward(tag), std::forward(args)...) - } noexcept; - }; - -} // namespace simdjson -#endif // SIMDJSON_SUPPORTS_CONCEPTS -#endif // SIMDJSON_CONCEPTS_H -/* end file simdjson/concepts.h */ -/* including simdjson/constevalutil.h: #include "simdjson/constevalutil.h" */ -/* begin file simdjson/constevalutil.h */ -#ifndef SIMDJSON_CONSTEVALUTIL_H -#define SIMDJSON_CONSTEVALUTIL_H - -#include -#include -#include - -namespace simdjson { -namespace constevalutil { -#if SIMDJSON_CONSTEVAL - -constexpr static std::array json_quotable_character = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; -constexpr static std::array control_chars = { - "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", - "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r", - "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014", - "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b", - "\\u001c", "\\u001d", "\\u001e", "\\u001f"}; -// unoptimized, meant for compile-time execution -consteval std::string consteval_to_quoted_escaped(std::string_view input) { - std::string out = "\""; - for (char c : input) { - if (json_quotable_character[uint8_t(c)]) { - if (c == '"') { - out.append("\\\""); - } else if (c == '\\') { - out.append("\\\\"); - } else { - std::string_view v = control_chars[uint8_t(c)]; - out.append(v); - } - } else { - out.push_back(c); - } - } - out.push_back('"'); - return out; -} - -consteval std::size_t padded_key_length(std::size_t length) { - return (length + 15) / 16 * 16; -} -#endif // SIMDJSON_CONSTEVAL - - -#if SIMDJSON_SUPPORTS_CONCEPTS -template -struct fixed_string { - constexpr fixed_string(const char (&str)[N]) { - for (std::size_t i = 0; i < N; ++i) { - data[i] = str[i]; - } - } - constexpr fixed_string(const unsigned char (&str)[N]) { - for (std::size_t i = 0; i < N; ++i) { - data[i] = static_cast(str[i]); - } - } - char data[N]; - constexpr std::string_view view() const { return {data, N - 1}; } - constexpr size_t size() const { return N ; } - constexpr operator std::string_view() const { return view(); } - constexpr char operator[](std::size_t index) const { return data[index]; } - constexpr bool operator==(const fixed_string& other) const { - if (N != other.size()) { - return false; - } - for (std::size_t i = 0; i < N; ++i) { - if (data[i] != other.data[i]) { - return false; - } - } - return true; - } -}; -template -fixed_string(const char (&)[N]) -> fixed_string; - -template -inline constexpr bool is_fixed_string = false; -template -inline constexpr bool is_fixed_string> = true; - -template -struct string_constant { - static constexpr std::string_view value = str.view(); -}; - -#if SIMDJSON_CONSTEVAL -template -consteval std::size_t static_string_size() { - return Text::text().size(); -} -template -consteval std::size_t padded_static_string_size() { - return padded_key_length(static_string_size()); -} - -template -consteval auto make_static_string() { - std::array()> out{}; - std::string text = Text::text(); - // The tail past text.size() stays zero: it is the padding. - for (std::size_t i = 0; i < text.size(); ++i) { - out[i] = text[i]; - } - return out; -} - -template -inline constexpr auto static_string = make_static_string(); - -template -struct quoted_escaped_text { - static consteval std::string text() { return consteval_to_quoted_escaped(str.view()); } -}; -#endif // SIMDJSON_CONSTEVAL -#endif // SIMDJSON_SUPPORTS_CONCEPTS - -} // namespace constevalutil -} // namespace simdjson -#endif // SIMDJSON_CONSTEVALUTIL_H -/* end file simdjson/constevalutil.h */ - -#if SIMDJSON_SUPPORTS_CHAR8_T -#include -#include -#endif - -/** - * @brief The top level simdjson namespace, containing everything the library provides. - */ -namespace simdjson { - -SIMDJSON_PUSH_DISABLE_UNUSED_WARNINGS - -/** The maximum document size supported by simdjson. */ -constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF; -/** The maximum depth of nested objects and arrays supported by simdjson. - A depth of SIMDJSON_MAXSIZE_BYTES/2 is not reasonable and would be - adversarial, but it serves as an upper bound for validation purposes. */ -constexpr size_t SIMDJSON_MAX_DEPTH = SIMDJSON_MAXSIZE_BYTES/2; - -/** - * The amount of padding needed in a buffer to parse JSON. - * - * The input buf should be readable up to buf + SIMDJSON_PADDING - * this is a stopgap; there should be a better description of the - * main loop and its behavior that abstracts over this - * See https://github.com/simdjson/simdjson/issues/174 - */ -constexpr size_t SIMDJSON_PADDING = 64; - -/** - * By default, simdjson supports this many nested objects and arrays. - * - * This is the default for parser::max_depth(). - */ -constexpr size_t DEFAULT_MAX_DEPTH = 1024; - -SIMDJSON_POP_DISABLE_UNUSED_WARNINGS - -class implementation; -struct padded_string; -class padded_string_view; -enum class stage1_mode; - -/** - * Stream format for parse_many/iterate_many. - */ -enum class stream_format { - whitespace_delimited, ///< Whitespace-delimited JSON documents (default, includes NDJSON/JSONL) - json_sequence, ///< RFC 7464 JSON text sequences (RS-delimited) - comma_delimited, ///< Comma-separated JSON documents (e.g., `{...},{...},{...}`) - comma_delimited_array,///< A single JSON array whose elements are iterated as - ///< comma-separated documents (e.g., `[{...},{...},{...}]`). - ///< The parser strips the outer `[` / `]` plus any - ///< surrounding JSON whitespace (space, tab, LF, CR) - ///< and then behaves like `comma_delimited` over the - ///< remaining bytes. - newline_delimited ///< NDJSON/JSON Lines where each document occupies exactly - ///< one line: documents are separated by line feeds and no - ///< document contains a raw line feed. Same inputs as - ///< `whitespace_delimited`, but the stronger guarantee lets - ///< the parser find the end of a document without walking - ///< it. On ondemand `iterate_many`, an unread remainder may - ///< be skipped by jumping to the next line feed without - ///< structure-validating that remainder. Use - ///< `whitespace_delimited` if unsure. -}; - -namespace internal { - -template -class atomic_ptr; -class dom_parser_implementation; -class escape_json_string; -class tape_ref; -struct value128; -enum class tape_type; - -#if SIMDJSON_SUPPORTS_CHAR8_T -/** - * Reinterpret a UTF-8 string as a C++20 std::u8string_view. No byte is copied - * or modified: char8_t and char have the same size, representation and - * alignment. Every string that simdjson produces is valid UTF-8, so this is a - * lossless view over the very same memory. - * @private - */ -simdjson_inline std::u8string_view as_u8string_view(std::string_view v) noexcept { - return std::u8string_view(reinterpret_cast(v.data()), v.size()); -} -#endif // SIMDJSON_SUPPORTS_CHAR8_T - -/** - * Assign a UTF-8 string to a string-like receiver. The general case simply - * assigns the std::string_view: it covers std::string and any user type that - * can be assigned from a std::string_view. - * @private - */ -template -simdjson_inline void assign_utf8(string_type &receiver, std::string_view content) noexcept { - receiver = content; -} - -#if SIMDJSON_SUPPORTS_CHAR8_T -/** - * Assign a UTF-8 string to a char8_t-based string (e.g., std::u8string). This - * overload is more specialized than the general one, so overload resolution - * prefers it whenever the receiver holds char8_t. - * @private - */ -template -simdjson_inline void assign_utf8(std::basic_string &receiver, std::string_view content) noexcept { - receiver.assign(reinterpret_cast(content.data()), content.size()); -} - -/** - * Assign a UTF-8 string to a char8_t-based string view (e.g., std::u8string_view). - * @private - */ -template -simdjson_inline void assign_utf8(std::basic_string_view &receiver, std::string_view content) noexcept { - receiver = std::basic_string_view(reinterpret_cast(content.data()), content.size()); -} -#endif // SIMDJSON_SUPPORTS_CHAR8_T - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_BASE_H -/* end file simdjson/base.h */ - -/* skipped duplicate #include "simdjson/error.h" */ -/* including simdjson/error-inl.h: #include "simdjson/error-inl.h" */ -/* begin file simdjson/error-inl.h */ -#ifndef SIMDJSON_ERROR_INL_H -#define SIMDJSON_ERROR_INL_H - -/* skipped duplicate #include "simdjson/error.h" */ - -#include - -namespace simdjson { - -inline bool is_fatal(error_code error) noexcept { - return error == TAPE_ERROR || error == INCOMPLETE_ARRAY_OR_OBJECT || error == OUT_OF_ORDER_ITERATION || error == DEPTH_ERROR; -} - -namespace internal { - // We store the error code so we can validate the error message is associated with the right code - struct error_code_info { - error_code code; - const char* message; // do not use a fancy std::string where a simple C string will do (no alloc, no destructor) - }; - // These MUST match the codes in error_code. We check this constraint in basictests. - extern SIMDJSON_DLLIMPORTEXPORT const error_code_info error_codes[]; -} // namespace internal - - -inline const char *error_message(error_code error) noexcept { - // If you're using error_code, we're trusting you got it from the enum. - return internal::error_codes[int(error)].message; -} - -// deprecated function -#ifndef SIMDJSON_DISABLE_DEPRECATED_API -inline const std::string error_message(int error) noexcept { - if (error < 0 || error >= error_code::NUM_ERROR_CODES) { - return internal::error_codes[UNEXPECTED_ERROR].message; - } - return internal::error_codes[error].message; -} -#endif // SIMDJSON_DISABLE_DEPRECATED_API - -inline std::ostream& operator<<(std::ostream& out, error_code error) noexcept { - return out << error_message(error); -} - -namespace internal { - -// -// internal::simdjson_result_base inline implementation -// - -template -simdjson_inline void simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_inline error_code simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_inline bool simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - - -template -simdjson_inline T& simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - - -template -simdjson_inline const T& simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T&& simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline simdjson_result_base::simdjson_result_base(T &&value, error_code error) noexcept - : std::pair(std::forward(value), error) {} -template -simdjson_inline simdjson_result_base::simdjson_result_base(error_code error) noexcept - : simdjson_result_base(T{}, error) {} -template -simdjson_inline simdjson_result_base::simdjson_result_base(T &&value) noexcept - : simdjson_result_base(std::forward(value), SUCCESS) {} -template -simdjson_inline simdjson_result_base::simdjson_result_base() noexcept - : simdjson_result_base(T{}, UNINITIALIZED) {} - -} // namespace internal - -/// -/// simdjson_result inline implementation -/// - -template -simdjson_inline void simdjson_result::tie(T &value, error_code &error) && noexcept { - std::forward>(*this).tie(value, error); -} - -template -simdjson_warn_unused simdjson_inline error_code -simdjson_result::get(T &value) && noexcept { - return std::forward>(*this).get(value); -} - -template -simdjson_inline error_code simdjson_result::error() const noexcept { - return internal::simdjson_result_base::error(); -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& simdjson_result::value() & noexcept(false) { - return internal::simdjson_result_base::value(); -} - -template -simdjson_inline T&& simdjson_result::value() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T&& simdjson_result::take_value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline simdjson_result::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& simdjson_result::value_unsafe() const& noexcept { - return internal::simdjson_result_base::value_unsafe(); -} - -template -simdjson_inline T&& simdjson_result::value_unsafe() && noexcept { - return std::forward>(*this).value_unsafe(); -} - -template -simdjson_inline simdjson_result::simdjson_result(T &&value, error_code error) noexcept - : internal::simdjson_result_base(std::forward(value), error) {} -template -simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept - : internal::simdjson_result_base(error) {} -template -simdjson_inline simdjson_result::simdjson_result(T &&value) noexcept - : internal::simdjson_result_base(std::forward(value)) {} -template -simdjson_inline simdjson_result::simdjson_result() noexcept - : internal::simdjson_result_base() {} - -} // namespace simdjson - -#endif // SIMDJSON_ERROR_INL_H -/* end file simdjson/error-inl.h */ -/* including simdjson/implementation.h: #include "simdjson/implementation.h" */ -/* begin file simdjson/implementation.h */ -#ifndef SIMDJSON_IMPLEMENTATION_H -#define SIMDJSON_IMPLEMENTATION_H - -/* including simdjson/internal/atomic_ptr.h: #include "simdjson/internal/atomic_ptr.h" */ -/* begin file simdjson/internal/atomic_ptr.h */ -#ifndef SIMDJSON_INTERNAL_ATOMIC_PTR_H -#define SIMDJSON_INTERNAL_ATOMIC_PTR_H - -/* skipped duplicate #include "simdjson/base.h" */ -#include - -namespace simdjson { -namespace internal { - -template -class atomic_ptr { -public: - atomic_ptr(T *_ptr) : ptr{_ptr} {} - - operator const T*() const { return ptr.load(); } - const T& operator*() const { return *ptr; } - const T* operator->() const { return ptr.load(); } - - operator T*() { return ptr.load(); } - T& operator*() { return *ptr; } - T* operator->() { return ptr.load(); } - atomic_ptr& operator=(T *_ptr) { ptr = _ptr; return *this; } - -private: - std::atomic ptr; -}; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_ATOMIC_PTR_H -/* end file simdjson/internal/atomic_ptr.h */ -/* including simdjson/internal/dom_parser_implementation.h: #include "simdjson/internal/dom_parser_implementation.h" */ -/* begin file simdjson/internal/dom_parser_implementation.h */ -#ifndef SIMDJSON_INTERNAL_DOM_PARSER_IMPLEMENTATION_H -#define SIMDJSON_INTERNAL_DOM_PARSER_IMPLEMENTATION_H - -/* skipped duplicate #include "simdjson/base.h" */ -/* skipped duplicate #include "simdjson/error.h" */ -#include - -namespace simdjson { - -namespace dom { -class document; -} // namespace dom - -/** -* This enum is used with the dom_parser_implementation::stage1 function. -* 1) The regular mode expects a fully formed JSON document. -* 2) The streaming_partial mode expects a possibly truncated -* input within a stream on JSON documents. -* 3) The stream_final mode allows us to truncate final -* unterminated strings. It is useful in conjunction with streaming_partial. -*/ -enum class stage1_mode { - regular, - streaming_partial, streaming_final, - json_sequence_partial, json_sequence_final, - comma_delimited_partial, comma_delimited_final -}; - -/** - * Returns true if mode == streaming_partial or mode == streaming_final - */ -inline bool is_streaming(stage1_mode mode) { - // performance note: it is probably faster to check that mode is different - // from regular than checking that it is either streaming_partial or streaming_final. - return (mode != stage1_mode::regular); - // return (mode == stage1_mode::streaming_partial || mode == stage1_mode::streaming_final); -} - -namespace internal { - - -/** - * An implementation of simdjson's DOM parser for a particular CPU architecture. - * - * This class is expected to be accessed only by pointer, and never move in memory (though the - * pointer can move). - */ -class dom_parser_implementation { -public: - - /** - * @private For internal implementation use - * - * Run a full JSON parse on a single document (stage1 + stage2). - * - * Guaranteed only to be called when capacity > document length. - * - * Overridden by each implementation. - * - * @param buf The json document to parse. *MUST* be allocated up to len + SIMDJSON_PADDING bytes. - * @param len The length of the json document. - * @return The error code, or SUCCESS if there was no error. - */ - simdjson_warn_unused virtual error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept = 0; - - /** - * @private For internal implementation use - * - * Stage 1 of the document parser. - * - * Guaranteed only to be called when capacity > document length. - * - * Overridden by each implementation. - * - * @param buf The json document to parse. - * @param len The length of the json document. - * @param streaming Whether this is being called by parser::parse_many. - * @return The error code, or SUCCESS if there was no error. - */ - simdjson_warn_unused virtual error_code stage1(const uint8_t *buf, size_t len, stage1_mode streaming) noexcept = 0; - - /** - * @private For internal implementation use - * - * Stage 2 of the document parser. - * - * Called after stage1(). - * - * Overridden by each implementation. - * - * @param doc The document to output to. - * @return The error code, or SUCCESS if there was no error. - */ - simdjson_warn_unused virtual error_code stage2(dom::document &doc) noexcept = 0; - - /** - * @private For internal implementation use - * - * Stage 2 of the document parser for parser::parse_many. - * - * Guaranteed only to be called after stage1(). - * Overridden by each implementation. - * - * @param doc The document to output to. - * @return The error code, SUCCESS if there was no error, or EMPTY if all documents have been parsed. - */ - simdjson_warn_unused virtual error_code stage2_next(dom::document &doc) noexcept = 0; - - /** @private A stage 2 entry point for parser::parse_many: see stage2_next_function. */ - typedef error_code (*stage2_next_function_t)(dom_parser_implementation &impl, dom::document &doc); - - /** - * @private For internal implementation use - * - * Get a function that does what stage2_next() does. dom::document_stream picks - * it once per batch and calls it for each document of the batch, so that the - * choice between the bounded and the unchecked walk costs nothing per document. - * - * Overridden by each implementation. - * - * @param bounded Whether the documents must be bounded to the structural - * indexes of the batch (see _bounded_batch). - */ - virtual stage2_next_function_t stage2_next_function(bool bounded) const noexcept = 0; - - /** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - * - * Overridden by each implementation. - * - * @param str pointer to the beginning of a valid UTF-8 JSON string, must end with an unescaped quote. - * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size. - * @param allow_replacement whether we allow a replacement character when the UTF-8 contains unmatched surrogate pairs. - * @return end of the of the written region (exclusive) or nullptr in case of error. - */ - simdjson_warn_unused virtual uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept = 0; - - /** - * Unescape a NON-valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_ptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - * - * Overridden by each implementation. - * - * @param str pointer to the beginning of a possibly invalid UTF-8 JSON string, must end with an unescaped quote. - * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size. - * @return end of the of the written region (exclusive) or nullptr in case of error. - */ - simdjson_warn_unused virtual uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept = 0; - - /** - * Change the capacity of this parser. - * - * The capacity can never exceed SIMDJSON_MAXSIZE_BYTES (e.g., 4 GB) - * and an CAPACITY error is returned if it is attempted. - * - * Generally used for reallocation. - * - * @param capacity The new capacity. - * @param max_depth The new max_depth. - * @return The error code, or SUCCESS if there was no error. - */ - virtual error_code set_capacity(size_t capacity) noexcept = 0; - - /** - * Change the max depth of this parser. - * - * Generally used for reallocation. - * - * @param capacity The new capacity. - * @param max_depth The new max_depth. - * @return The error code, or SUCCESS if there was no error. - */ - virtual error_code set_max_depth(size_t max_depth) noexcept = 0; - - /** - * Deallocate this parser. - */ - virtual ~dom_parser_implementation() = default; - - /** Number of structural indices passed from stage 1 to stage 2 */ - uint32_t n_structural_indexes{0}; - /** Structural indices passed from stage 1 to stage 2 */ - std::unique_ptr structural_indexes{}; - /** Next structural index to parse */ - uint32_t next_structural_index{0}; - - /** - * The largest document this parser can support without reallocating. - * - * @return Current capacity, in bytes. - */ - simdjson_pure simdjson_inline size_t capacity() const noexcept; - - /** - * The maximum level of nested object and arrays supported by this parser. - * - * @return Maximum depth, in bytes. - */ - simdjson_pure simdjson_inline size_t max_depth() const noexcept; - - /** - * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length - * and `max_depth` depth. - * - * @param capacity The new capacity. - * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH. - * @return The error, if there is one. - */ - simdjson_warn_unused inline error_code allocate(size_t capacity, size_t max_depth) noexcept; - - -protected: - /** - * The maximum document length this parser supports. - * - * Buffers are large enough to handle any document up to this length. - */ - size_t _capacity{0}; - - /** - * The maximum depth (number of nested objects and arrays) supported by this parser. - * - * Defaults to DEFAULT_MAX_DEPTH. - */ - size_t _max_depth{0}; - -public: - /** Whether to store big integers as strings instead of returning BIGINT_ERROR */ - bool _number_as_string{false}; - - /** - * Whether the input buffer passed to parse() is *not* padded to len + - * SIMDJSON_PADDING bytes. When true, stage 2 string parsing avoids reading - * past buf+len (it finishes the final, near-the-end bytes from a small padded - * scratch buffer). This is set only by the no-padding DOM parse entry points - * (dom::parser::parse_unpadded); the default padded fast path leaves it false - * and is unaffected. - */ - bool _unpadded{false}; - - /** - * Whether stage 2 must bound the documents of the batch indexed by the last - * stage 1 run to its structural indexes (dom::document_stream sets it, see - * document_stream::run_stage1, and passes it to stage2_next_function). Other - * batches keep the unchecked walk. - */ - bool _bounded_batch{false}; - -protected: - - // Declaring these so that subclasses can use them to implement their constructors. - simdjson_inline dom_parser_implementation() noexcept; - simdjson_inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - simdjson_inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - - simdjson_inline dom_parser_implementation(const dom_parser_implementation &) noexcept = delete; - simdjson_inline dom_parser_implementation &operator=(const dom_parser_implementation &other) noexcept = delete; -}; // class dom_parser_implementation - -simdjson_inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -simdjson_inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -simdjson_inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -simdjson_pure simdjson_inline size_t dom_parser_implementation::capacity() const noexcept { - return _capacity; -} - -simdjson_pure simdjson_inline size_t dom_parser_implementation::max_depth() const noexcept { - return _max_depth; -} - -simdjson_warn_unused -inline error_code dom_parser_implementation::allocate(size_t capacity, size_t max_depth) noexcept { - if (this->max_depth() != max_depth) { - error_code err = set_max_depth(max_depth); - if (err) { return err; } - } - if (_capacity != capacity) { - error_code err = set_capacity(capacity); - if (err) { return err; } - } - return SUCCESS; -} - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/internal/dom_parser_implementation.h */ - -#include - -namespace simdjson { - -/** - * Validate the UTF-8 string. - * - * @param buf the string to validate. - * @param len the length of the string in bytes. - * @return true if the string is valid UTF-8. - */ -simdjson_warn_unused bool validate_utf8(const char * buf, size_t len) noexcept; -/** - * Validate the UTF-8 string. - * - * @param sv the string_view to validate. - * @return true if the string is valid UTF-8. - */ -simdjson_inline simdjson_warn_unused bool validate_utf8(const std::string_view sv) noexcept { - return validate_utf8(sv.data(), sv.size()); -} - -/** - * Validate the UTF-8 string. - * - * @param p the string to validate. - * @return true if the string is valid UTF-8. - */ -simdjson_inline simdjson_warn_unused bool validate_utf8(const std::string& s) noexcept { - return validate_utf8(s.data(), s.size()); -} - -/** - * An implementation of simdjson for a particular CPU architecture. - * - * Also used to maintain the currently active implementation. The active implementation is - * automatically initialized on first use to the most advanced implementation supported by the host. - */ -class implementation { -public: - - /** - * The name of this implementation. - * - * const implementation *impl = simdjson::get_active_implementation(); - * cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl; - * - * @return the name of the implementation, e.g. "haswell", "westmere", "arm64". - */ - virtual std::string name() const { return std::string(_name); } - - /** - * The description of this implementation. - * - * const implementation *impl = simdjson::get_active_implementation(); - * cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl; - * - * @return the description of the implementation, e.g. "Intel/AMD AVX2", "Intel/AMD SSE4.2", "ARM NEON". - */ - virtual std::string description() const { return std::string(_description); } - - /** - * The instruction sets this implementation is compiled against - * and the current CPU match. This function may poll the current CPU/system - * and should therefore not be called too often if performance is a concern. - * - * @return true if the implementation can be safely used on the current system (determined at runtime). - */ - bool supported_by_runtime_system() const; - - /** - * @private For internal implementation use - * - * The instruction sets this implementation is compiled against. - * - * @return a mask of all required `internal::instruction_set::` values. - */ - virtual uint32_t required_instruction_sets() const { return _required_instruction_sets; } - - /** - * @private For internal implementation use - * - * const implementation *impl = simdjson::get_active_implementation(); - * cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl; - * - * @param capacity The largest document that will be passed to the parser. - * @param max_depth The maximum JSON object/array nesting this parser is expected to handle. - * @param dst The place to put the resulting parser implementation. - * @return the error code, or SUCCESS if there was no error. - */ - virtual error_code create_dom_parser_implementation( - size_t capacity, - size_t max_depth, - std::unique_ptr &dst - ) const noexcept = 0; - - /** - * @private For internal implementation use - * - * Minify the input string assuming that it represents a JSON string, does not parse or validate. - * - * Overridden by each implementation. - * - * @param buf the json document to minify. - * @param len the length of the json document. - * @param dst the buffer to write the minified document to. *MUST* be allocated up to len + SIMDJSON_PADDING bytes. - * @param dst_len the number of bytes written. Output only. - * @return the error code, or SUCCESS if there was no error. - */ - simdjson_warn_unused virtual error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept = 0; - - - /** - * Validate the UTF-8 string. - * - * Overridden by each implementation. - * - * @param buf the string to validate. - * @param len the length of the string in bytes. - * @return true if and only if the string is valid UTF-8. - */ - simdjson_warn_unused virtual bool validate_utf8(const char *buf, size_t len) const noexcept = 0; - -protected: - /** @private Construct an implementation with the given name and description. For subclasses. */ - simdjson_inline implementation( - std::string_view name, - std::string_view description, - uint32_t required_instruction_sets - ) : - _name(name), - _description(description), - _required_instruction_sets(required_instruction_sets) - { - } -protected: - ~implementation() = default; - -private: - /** - * The name of this implementation. - */ - std::string_view _name; - - /** - * The description of this implementation. - */ - std::string_view _description; - - /** - * Instruction sets required for this implementation. - */ - const uint32_t _required_instruction_sets; -}; - -/** @private */ -namespace internal { - -/** - * The list of available implementations compiled into simdjson. - */ -class available_implementation_list { -public: - /** Get the list of available implementations compiled into simdjson */ - simdjson_inline available_implementation_list() {} - /** Number of implementations */ - size_t size() const noexcept; - /** STL const begin() iterator */ - const implementation * const *begin() const noexcept; - /** STL const end() iterator */ - const implementation * const *end() const noexcept; - - /** - * Get the implementation with the given name. - * - * Case sensitive. - * - * const implementation *impl = simdjson::get_available_implementations()["westmere"]; - * if (!impl) { exit(1); } - * if (!imp->supported_by_runtime_system()) { exit(1); } - * simdjson::get_active_implementation() = impl; - * - * @param name the implementation to find, e.g. "westmere", "haswell", "arm64" - * @return the implementation, or nullptr if the parse failed. - */ - const implementation * operator[](const std::string_view &name) const noexcept { - for (const implementation * impl : *this) { - if (impl->name() == name) { return impl; } - } - return nullptr; - } - - /** - * Detect the most advanced implementation supported by the current host. - * - * This is used to initialize the implementation on startup. - * - * const implementation *impl = simdjson::available_implementation::detect_best_supported(); - * simdjson::get_active_implementation() = impl; - * - * @return the most advanced supported implementation for the current host, or an - * implementation that returns UNSUPPORTED_ARCHITECTURE if there is no supported - * implementation. Will never return nullptr. - */ - const implementation *detect_best_supported() const noexcept; -}; - -} // namespace internal - -/** - * The list of available implementations compiled into simdjson. - */ -extern SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list& get_available_implementations(); - -/** - * The active implementation. - * - * Automatically initialized on first use to the most advanced implementation supported by this hardware. - */ -extern SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr& get_active_implementation(); - -} // namespace simdjson - -#endif // SIMDJSON_IMPLEMENTATION_H -/* end file simdjson/implementation.h */ -/* including simdjson/minify.h: #include "simdjson/minify.h" */ -/* begin file simdjson/minify.h */ -#ifndef SIMDJSON_MINIFY_H -#define SIMDJSON_MINIFY_H - -/* skipped duplicate #include "simdjson/base.h" */ -/* including simdjson/padded_string.h: #include "simdjson/padded_string.h" */ -/* begin file simdjson/padded_string.h */ -#ifndef SIMDJSON_PADDED_STRING_H -#define SIMDJSON_PADDED_STRING_H - -/* skipped duplicate #include "simdjson/base.h" */ -/* skipped duplicate #include "simdjson/error.h" */ - -/* skipped duplicate #include "simdjson/error-inl.h" */ - -#include -#include -#include -#include - -namespace simdjson { - -class padded_string_view; - -/** - * String with extra allocation for ease of use with parser::parse() - * - * This is a move-only class, it cannot be copied. - */ -struct padded_string final { - - /** - * Create a new, empty padded string. - */ - explicit inline padded_string() noexcept; - /** - * Create a new padded string buffer. - * - * @param length the size of the string. - */ - explicit inline padded_string(size_t length) noexcept; - /** - * Create a new padded string by copying the given input. - * - * @param data the buffer to copy - * @param length the number of bytes to copy - */ - explicit inline padded_string(const char *data, size_t length) noexcept; -#ifdef __cpp_char8_t - explicit inline padded_string(const char8_t *data, size_t length) noexcept; -#endif - /** - * Create a new padded string by copying the given input. - * - * @param str_ the string to copy - */ - inline padded_string(const std::string & str_ ) noexcept; - /** - * Create a new padded string by copying the given input. - * - * @param sv_ the string to copy - */ - inline padded_string(std::string_view sv_) noexcept; - /** - * Move one padded string into another. - * - * The original padded string will be reduced to zero capacity. - * - * @param o the string to move. - */ - inline padded_string(padded_string &&o) noexcept; - /** - * Move one padded string into another. - * - * The original padded string will be reduced to zero capacity. - * - * @param o the string to move. - */ - inline padded_string &operator=(padded_string &&o) noexcept; - inline void swap(padded_string &o) noexcept; - ~padded_string() noexcept; - - /** - * The length of the string. - * - * Does not include padding. - */ - size_t size() const noexcept; - - /** - * The length of the string. - * - * Does not include padding. - */ - size_t length() const noexcept; - - /** - * The string data. - **/ - const char *data() const noexcept; - const uint8_t *u8data() const noexcept { return static_cast(static_cast(data_ptr));} - - /** - * The string data. - **/ - char *data() noexcept; - - /** - * Append data to the padded string. Return true on success, false on failure. - * The complexity is O(n) where n is the new size of the string. If you are - * doing multiple appends, consider using padded_string_builder for better performance. - * - * @param data the buffer to append - * @param length the number of bytes to append - */ - inline bool append(const char *data, size_t length) noexcept; - - /** - * Create a std::string_view with the same content. - */ - operator std::string_view() const; - - /** - * Create a padded_string_view with the same content. - */ - operator padded_string_view() const noexcept; - - /** - * Load this padded string from a file. - * - * ## Windows and Unicode - * - * Windows users who need to read files with non-ANSI characters in the - * name should set their code page to UTF-8 (65001) before calling this - * function. This should be the default with Windows 11 and better. - * Further, they may use the AreFileApisANSI function to determine whether - * the filename is interpreted using the ANSI or the system default OEM - * codepage, and they may call SetFileApisToOEM accordingly. - * - * @return IO_ERROR on error. Be mindful that on some 32-bit systems, - * the file size might be limited to 2 GB. - * - * @param path the path to the file. - **/ - inline static simdjson_result load(std::string_view path) noexcept; - - #if defined(_WIN32) && SIMDJSON_CPLUSPLUS17 - /** - * This function accepts a wide string path (UTF-16) and converts it to - * UTF-8 before loading the file. This allows windows users to work - * with unicode file paths without manually converting the paths every time. - * - * @return IO_ERROR on error, including conversion failures. - * - * @param path the path to the file as a wide string. - **/ - inline static simdjson_result load(std::wstring_view path) noexcept; - #endif - -private: - friend class padded_string_builder; - padded_string &operator=(const padded_string &o) = delete; - padded_string(const padded_string &o) = delete; - - size_t viable_size{0}; - char *data_ptr{nullptr}; - -}; // padded_string - -/** - * Builder for constructing padded_string incrementally. - * - * This class allows efficient appending of data and then building a padded_string. - */ -class padded_string_builder { -public: - /** - * Create a new, empty padded string builder. - */ - inline padded_string_builder() noexcept; - - /** - * Create a new padded string builder with initial capacity. - * - * @param capacity the initial capacity of the builder. - */ - inline padded_string_builder(size_t capacity) noexcept; - - /** - * Move constructor. - */ - inline padded_string_builder(padded_string_builder &&o) noexcept; - - /** - * Move assignment. - */ - inline padded_string_builder &operator=(padded_string_builder &&o) noexcept; - - /** - * Copy constructor (deleted). - */ - padded_string_builder(const padded_string_builder &) = delete; - - /** - * Copy assignment (deleted). - */ - padded_string_builder &operator=(const padded_string_builder &) = delete; - - /** - * Destructor. - */ - inline ~padded_string_builder() noexcept; - - /** - * Append data to the builder. - * - * @param newdata the buffer to append - * @param length the number of bytes to append - * @return true if the append succeeded, false if allocation failed - */ - inline bool append(const char *newdata, size_t length) noexcept; - - /** - * Append a string view to the builder. - * - * @param sv the string view to append - * @return true if the append succeeded, false if allocation failed - */ - inline bool append(std::string_view sv) noexcept; - - /** - * Get the current length of the built string. - */ - inline size_t length() const noexcept; - - /** - * Build a padded_string from the current content. The builder's content - * is not modified. If you want to avoid the copy, use convert() instead. - * - * @return a padded_string containing a copy of the built content. - */ - inline padded_string build() const noexcept; - - /** - * Convert the current content into a padded_string. The - * builder's content is emptied, the capacity is lost. - * - * @return a padded_string containing the built content. - */ - inline padded_string convert() noexcept; -private: - size_t size{0}; - size_t capacity{0}; - char *data{nullptr}; - - /** - * Ensure the builder has enough capacity. - * - * @param additional the additional capacity needed. - * @return true if the reservation succeeded, false if allocation failed - */ - inline bool reserve(size_t additional) noexcept; -}; - -/** - * Send padded_string instance to an output stream. - * - * @param out The output stream. - * @param s The padded_string instance. - * @throw if there is an error with the underlying output stream. simdjson itself will not throw. - */ -inline std::ostream& operator<<(std::ostream& out, const padded_string& s) { return out << s.data(); } - -#if SIMDJSON_EXCEPTIONS -/** - * Send padded_string instance to an output stream. - * - * @param out The output stream. - * @param s The padded_string instance. - * @throw simdjson_error if the result being printed has an error. If there is an error with the - * underlying output stream, that error will be propagated (simdjson_error will not be - * thrown). - */ -inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false) { return out << s.value(); } -#endif - -#if SIMDJSON_HAS_PADDED_MEMORY_MAP -/** - * A class representing a memory-mapped file with padding. - * - * On POSIX systems (Linux, macOS, BSD, ...), this uses `mmap` to map the file - * contents directly into memory, which is efficient for large files (no copy). - * - * On Windows, this class is disabled by default and must be opted into at - * build time by defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`. When - * enabled, `` must also be included before `` and - * the compilation must target Windows 10, version 1803 or later. The - * Windows implementation uses the modern memory APIs (`VirtualAlloc2`, - * `CreateFileMapping2`, `MapViewOfFile3`) with the placeholder virtual - * memory mechanism to always achieve true zero-copy mapping with - * contiguous zero-filled padding. - * - * Either way, the resulting `padded_string_view` carries at least - * `SIMDJSON_PADDING` bytes of accessible zero-filled padding after the file - * content, so it can be consumed directly by the simdjson parsers (including - * `parse_many` / `iterate_many`). - */ -class padded_memory_map { -public: - /** - * Create a new padded memory map for the given file. - * After creating the memory map, you can call view() to get a padded_string_view of the file content. - * The memory map will be automatically released when the padded_memory_map instance is destroyed. - * On POSIX systems, the file content is not copied, so this is efficient for large files. - * On Windows, the file is mapped into memory via `MapViewOfFile3` (zero-copy). - * In all cases, the file must remain unchanged while the memory map is in use. - * In case of error (e.g., file not found, permission denied, etc.), the memory map will be - * invalid and view() will return an empty view. - * You can check if the memory map is valid by calling is_valid() before using view(). - * - * @param filename the path to the file to memory-map. - */ - simdjson_inline padded_memory_map(const char *filename) noexcept; - /** - * Destroy the padded memory map and release any resources. - */ - simdjson_inline ~padded_memory_map() noexcept; - - // lifetime of the view is tied to the memory map, so we can return a view - // directly - /** - * Get a view of the memory-mapped file. It always succeeds, but the view may be empty - * if the memory map is invalid (e.g., due to file not found, permission denied, etc.). - * You can check if the memory map is valid by calling is_valid() before using the view. - * - * Lifetime of the view is tied to the memory map, so the view should not be used after the - * padded_memory_map instance is destroyed. - * - * @return a padded_string_view representing the memory-mapped file, or an empty view if the memory map is invalid. - */ - simdjson_inline simdjson::padded_string_view view() const noexcept simdjson_lifetime_bound; - /** - * Check if the memory map is valid. - * - * @return true if the memory map is valid, false otherwise. - */ - simdjson_inline bool is_valid() const noexcept; - -private: - padded_memory_map() = delete; - padded_memory_map(const padded_memory_map &) = delete; - padded_memory_map &operator=(const padded_memory_map &) = delete; - const char *data{nullptr}; - size_t size{0}; -#ifdef _WIN32 - // When the file ends near an allocation-granularity boundary, we use the - // placeholder API to append zero-filled padding pages. This pointer tracks - // that region so the destructor can release it with VirtualFree. - void *padding_view_{nullptr}; -#endif -}; -#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP - - - -} // namespace simdjson - -// This is deliberately outside of simdjson so that people get it without having to use the namespace -inline simdjson::padded_string operator ""_padded(const char *str, size_t len); -#ifdef __cpp_char8_t -inline simdjson::padded_string operator ""_padded(const char8_t *str, size_t len); -#endif - -namespace simdjson { -namespace internal { - -// The allocate_padded_buffer function is a low-level function to allocate memory -// with padding so we can read past the "length" bytes safely. It is used by -// the padded_string class automatically. It returns nullptr in case -// of error: the caller should check for a null pointer. -// The length parameter is the maximum size in bytes of the string. -// The caller is responsible to free the memory (e.g., delete[] (...)). -inline char *allocate_padded_buffer(size_t length) noexcept; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_PADDED_STRING_H -/* end file simdjson/padded_string.h */ -#include -#include -#include - -namespace simdjson { - -/** - * - * Minify the input string assuming that it represents a JSON string, does not parse or validate. - * This function is much faster than parsing a JSON string and then writing a minified version of it. - * However, it does not validate the input. It will merely return an error in simple cases (e.g., if - * there is a string that was never terminated). - * - * - * @param buf the json document to minify. - * @param len the length of the json document. - * @param dst the buffer to write the minified document to. *MUST* be allocated up to len bytes. - * @param dst_len the number of bytes written. Output only. - * @return the error code, or SUCCESS if there was no error. - */ -simdjson_warn_unused error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept; - -} // namespace simdjson - -#endif // SIMDJSON_MINIFY_H -/* end file simdjson/minify.h */ -/* skipped duplicate #include "simdjson/padded_string.h" */ -/* including simdjson/padded_string-inl.h: #include "simdjson/padded_string-inl.h" */ -/* begin file simdjson/padded_string-inl.h */ -#ifndef SIMDJSON_PADDED_STRING_INL_H -#define SIMDJSON_PADDED_STRING_INL_H - -/* skipped duplicate #include "simdjson/padded_string.h" */ -/* including simdjson/padded_string_view.h: #include "simdjson/padded_string_view.h" */ -/* begin file simdjson/padded_string_view.h */ -#ifndef SIMDJSON_PADDED_STRING_VIEW_H -#define SIMDJSON_PADDED_STRING_VIEW_H - -/* skipped duplicate #include "simdjson/portability.h" */ -/* skipped duplicate #include "simdjson/base.h" // for SIMDJSON_PADDING */ -/* skipped duplicate #include "simdjson/error.h" */ - -#include -#include -#include -#include -#if SIMDJSON_CPLUSPLUS17 -#include -#endif - -namespace simdjson { - -/** - * User-provided string that promises it has extra padded bytes at the end for use with parser::parse(). - */ -class padded_string_view : public std::string_view { -private: - size_t _capacity{0}; - -public: - /** Create an empty padded_string_view. */ - inline padded_string_view() noexcept = default; - - /** - * Promise the given buffer has at least SIMDJSON_PADDING extra bytes allocated to it. - * - * @param s The string. - * @param len The length of the string (not including padding). - * @param capacity The allocated length of the string, including padding. If the capacity is less - * than the length, the capacity will be set to the length. - */ - explicit inline padded_string_view(const char* s, size_t len, size_t capacity) noexcept; - /** overload explicit inline padded_string_view(const char* s, size_t len) noexcept */ - explicit inline padded_string_view(const uint8_t* s, size_t len, size_t capacity) noexcept; -#ifdef __cpp_char8_t - explicit inline padded_string_view(const char8_t* s, size_t len, size_t capacity) noexcept; -#endif - /** - * Promise the given string has at least SIMDJSON_PADDING extra bytes allocated to it. - * - * The capacity of the string will be used to determine its padding. - * - * @param s The string. - */ - explicit inline padded_string_view(const std::string &s) noexcept; - - /** - * Promise the given string_view has at least SIMDJSON_PADDING extra bytes allocated to it. - * - * @param s The string. - * @param capacity The allocated length of the string, including padding. If the capacity is less - * than the length, the capacity will be set to the length. - */ - explicit inline padded_string_view(std::string_view s, size_t capacity) noexcept; - - /** The number of allocated bytes. */ - inline size_t capacity() const noexcept; - - /** - * Check that the view has sufficient padding: either SIMDJSON_PADDING bytes of - * capacity past the end of the document, or enough trailing white space in - * the document to stand in for the missing bytes. - */ - inline bool has_sufficient_padding() const noexcept; - - /** - * The parser may read up to SIMDJSON_PADDING bytes past the end of the - * document. When the capacity falls short of that but the document ends - * with enough white space to make up for it (see has_sufficient_padding()), - * this moves that white space out of the document and into the padding, so - * that padding() >= SIMDJSON_PADDING. It does nothing otherwise. - * - * @return whether the document was shortened - */ - inline bool use_trailing_whitespace_as_padding() noexcept; - - /** - * Remove the UTF-8 Byte Order Mark (BOM) if it exists. - * - * @return whether a BOM was found and removed - */ - inline bool remove_utf8_bom() noexcept; - - /** The amount of padding on the string (capacity() - length()) */ - inline size_t padding() const noexcept; - -}; // padded_string_view - -/** - * Get the system's memory page size. By default, we return - * 4096 bytes, which is the most common page size. On systems - * where the page size is not a multiple of 4096 bytes, and not - * a unix-like system, nor Windows, this function may return an - * incorrect value. - * - * @return The page size in bytes. - */ -inline uint32_t get_page_size() noexcept; - -#if SIMDJSON_CPLUSPLUS17 -/** - * A padded_input is a wrapper around either a padded_string_view or a padded_string. - * It will automatically pad a string_view if it does not have sufficient padding - * up to the end of the memory page. Note that a requirement for this method to - * make sense is to be on a system with a page size of at least 4096 (which is - * universal except on some embedded systems). - */ -struct padded_input { - /** - * Construct a padded_input from a string_view. If the string_view does not have sufficient padding, - * the data will be copied into a padded_string and the padded_string_view will point to the - * padded_string's data. Otherwise, the padded_string_view will point to the original string_view's data. - */ - inline explicit padded_input(std::string_view sv); - /** - * Construct a padded_input from a C-style string (length specified). If the string does not have sufficient padding, - * the data will be copied into a padded_string and the padded_string_view will point to the - * padded_string's data. Otherwise, the padded_string_view will point to the original string's data. - */ - inline explicit padded_input(const char *data, size_t length); - /** - * Construct a padded_input from a std::string. If the string does not have sufficient padding - * (considering its capacity), the data will be copied into a padded_string and the padded_string_view - * will point to the padded_string's data. Otherwise, the padded_string_view will point to the - * original string's data. - */ - inline explicit padded_input(const std::string &s); - - /** - * Check if the padded_input is a view. - * - * @return true if the padded_input is a view, false otherwise. - */ - inline bool is_view() const noexcept; - - /** - * Convert the padded_input to a padded_string_view. - * - * @return The padded_string_view. - */ - inline operator simdjson::padded_string_view() const noexcept; - -private: - std::variant storage; - // whether we cross a page boundary and need to allocate a new padded string. - static inline bool needs_allocation(const char* buf, size_t len, size_t padding = SIMDJSON_PADDING) noexcept; -}; - -#endif // SIMDJSON_CPLUSPLUS17 - -#if SIMDJSON_EXCEPTIONS -/** - * Send padded_string instance to an output stream. - * - * @param out The output stream. - * @param s The padded_string_view. - * @throw simdjson_error if the result being printed has an error. If there is an error with the - * underlying output stream, that error will be propagated (simdjson_error will not be - * thrown). - */ -inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false); -#endif - -/** - * Create a padded_string_view from a string. The string will be padded with up to SIMDJSON_PADDING - * space characters. The resulting padded_string_view will have a length equal to the original - * string, except maybe for trailing white space characters. - * - * @param s The string. - * @return The padded string. - */ -inline padded_string_view pad(std::string& s) noexcept; - -/** - * Create a padded_string_view from a string. The capacity of the string will be padded with SIMDJSON_PADDING - * characters. The resulting padded_string_view will have a length equal to the original - * string. - * - * @param s The string. - * @return The padded string. - */ -inline padded_string_view pad_with_reserve(std::string& s) noexcept; - -/** - * Return the index-th document-aligned slice of a delimited stream. - * - * The input is divided into blocks of block_size bytes and each boundary is - * moved forward to just past the next delimiter, so a document is never split. - * The delimiter must not occur inside a document: a line feed for NDJSON, a - * record separator (0x1E) for RFC 7464. - * - * Slices are contiguous and non-overlapping, and each may be parsed - * independently, so callers can process them on as many threads as they like. - * - * Iterate while index * block_size < data.size(). A slice is empty when its - * block falls entirely inside one document, which happens only if that document - * is longer than block_size; skip it and continue. With block_size larger than - * the longest document, no slice is ever empty. - * - * @param data The padded input. - * @param delimiter The byte that separates documents. - * @param block_size The nominal slice size, before snapping. - * @param index Which slice to return, counting from zero. - */ -inline padded_string_view slice_at(padded_string_view data, char delimiter, - size_t block_size, size_t index) noexcept; - -} // namespace simdjson - -#endif // SIMDJSON_PADDED_STRING_VIEW_H -/* end file simdjson/padded_string_view.h */ - -/* skipped duplicate #include "simdjson/error-inl.h" */ -/* including simdjson/padded_string_view-inl.h: #include "simdjson/padded_string_view-inl.h" */ -/* begin file simdjson/padded_string_view-inl.h */ -#ifndef SIMDJSON_PADDED_STRING_VIEW_INL_H -#define SIMDJSON_PADDED_STRING_VIEW_INL_H - -/* skipped duplicate #include "simdjson/padded_string_view.h" */ -/* skipped duplicate #include "simdjson/error-inl.h" */ - -#include /* memcmp */ - -// for page size computation. -#if SIMDJSON_HAS_UNISTD_H - #include - #if defined(__APPLE__) - #include - #endif -#endif - - -namespace simdjson { - -inline padded_string_view::padded_string_view(const char* s, size_t len, size_t capacity) noexcept - : std::string_view(s, len), _capacity(capacity) -{ - if(_capacity < len) { _capacity = len; } -} - -inline padded_string_view::padded_string_view(const uint8_t* s, size_t len, size_t capacity) noexcept - : padded_string_view(reinterpret_cast(s), len, capacity) -{ -} -#ifdef __cpp_char8_t -inline padded_string_view::padded_string_view(const char8_t* s, size_t len, size_t capacity) noexcept - : padded_string_view(reinterpret_cast(s), len, capacity) -{ -} -#endif -inline padded_string_view::padded_string_view(const std::string &s) noexcept - : std::string_view(s), _capacity(s.capacity()) -{ -} - -inline padded_string_view::padded_string_view(std::string_view s, size_t capacity) noexcept - : std::string_view(s), _capacity(capacity) -{ - if(_capacity < s.length()) { _capacity = s.length(); } -} - -inline bool padded_string_view::has_sufficient_padding() const noexcept { - if (padding() >= SIMDJSON_PADDING) { - return true; - } - size_t missing_padding = SIMDJSON_PADDING - padding(); - if(length() < missing_padding) { return false; } - - for (size_t i = length() - missing_padding; i < length(); i++) { - char c = data()[i]; - if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { - return false; - } - } - return true; -} - -inline bool padded_string_view::use_trailing_whitespace_as_padding() noexcept { - if (padding() >= SIMDJSON_PADDING || !has_sufficient_padding()) { - return false; - } - remove_suffix(SIMDJSON_PADDING - padding()); - return true; -} - -inline size_t padded_string_view::capacity() const noexcept { return _capacity; } - -inline size_t padded_string_view::padding() const noexcept { return capacity() - length(); } - -inline bool padded_string_view::remove_utf8_bom() noexcept { - if(length() < 3) { return false; } - if (std::memcmp(data(), "\xEF\xBB\xBF", 3) == 0) { - remove_prefix(3); - _capacity -= 3; - return true; - } - return false; -} - -#if SIMDJSON_EXCEPTIONS -inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false) { return out << s.value(); } -#endif - -inline padded_string_view pad(std::string& s) noexcept { - size_t existing_padding = 0; - for (size_t i = s.size(); i > 0; i--) { - char c = s[i - 1]; - if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { - existing_padding++; - } else { - break; - } - } - size_t needed_padding = 0; - if (existing_padding < SIMDJSON_PADDING) { - needed_padding = SIMDJSON_PADDING - existing_padding; - s.append(needed_padding, ' '); - } - - return padded_string_view(s.data(), s.size() - needed_padding, s.size()); -} - -inline padded_string_view pad_with_reserve(std::string& s) noexcept { - if (s.capacity() - s.size() < SIMDJSON_PADDING) { - s.reserve(s.size() + SIMDJSON_PADDING ); - } - return padded_string_view(s.data(), s.size(), s.capacity()); -} - -inline uint32_t get_page_size() noexcept { -#if defined(_WINDOWS_) // if and only if someone loaded Windows.h, we can get the page size from there. -// Otherwise, we assume 4096. - static const uint32_t cached = []() -> uint32_t { - SYSTEM_INFO si; - GetSystemInfo(&si); - return static_cast(si.dwPageSize); - }(); - return cached; -#elif SIMDJSON_HAS_UNISTD_H - static const uint32_t cached = []() -> uint32_t { - long page_size = sysconf(_SC_PAGESIZE); - if (page_size > 0) { - return static_cast(page_size); - } - return 4096; // fallback - }(); - return cached; -#else - return 4096; // fallback -#endif -} -#if SIMDJSON_CPLUSPLUS17 - -inline padded_input::padded_input(std::string_view sv) - : storage(simdjson::padded_string_view{}) { - if (needs_allocation(sv.data(), sv.size())) { - storage = simdjson::padded_string(sv); - } else { - storage = simdjson::padded_string_view( - sv.data(), sv.size(), sv.size() + simdjson::SIMDJSON_PADDING); - } -} - -inline padded_input::padded_input(const char *data, size_t length) - : storage(simdjson::padded_string_view{}) { - if (needs_allocation(data, length)) { - storage = simdjson::padded_string(data, length); - } else { - storage = simdjson::padded_string_view( - data, length, length + simdjson::SIMDJSON_PADDING); - } -} - -inline padded_input::padded_input(const std::string &s) - : storage(simdjson::padded_string_view{}) { - const size_t len = s.size(); - const size_t cap = s.capacity(); - // Here we have the string content from data() to data() + size(), - // but the memory is accessible from data() to data() + capacity(). - const size_t needed_padding = (cap - len) < simdjson::SIMDJSON_PADDING - ? simdjson::SIMDJSON_PADDING - (cap - len) : 0; - if (needed_padding > 0 && needs_allocation(s.data(), cap, needed_padding)) { - storage = simdjson::padded_string(s); - } else { - storage = simdjson::padded_string_view( - s.data(), len, len + simdjson::SIMDJSON_PADDING); - } -} - -inline bool padded_input::is_view() const noexcept { - return std::holds_alternative(storage); -} - -inline padded_input::operator simdjson::padded_string_view() const noexcept { - return std::visit([](const auto& p) -> simdjson::padded_string_view { - return p; - }, storage); -} - -inline bool padded_input::needs_allocation(const char* buf, size_t len, size_t padding) noexcept { - if(len == 0) { return false; } - const auto page_size = get_page_size(); - return ((reinterpret_cast(buf + len - 1) % page_size) - + padding >= static_cast(page_size)); -} -#endif // SIMDJSON_CPLUSPLUS17 - -inline padded_string_view slice_at(padded_string_view data, char delimiter, - size_t block_size, size_t index) noexcept { - if (block_size == 0 || index > data.size() / block_size) { return {}; } - const size_t raw_begin = index * block_size; - if (raw_begin >= data.size()) { return {}; } - - auto snap = [&](size_t want) -> size_t { - if (want >= data.size()) { return data.size(); } - const void *p = std::memchr(data.data() + want, delimiter, data.size() - want); - return p ? size_t(static_cast(p) - data.data()) + 1 : data.size(); - }; - - const size_t begin = (raw_begin == 0) ? 0 : snap(raw_begin); - const size_t end = snap(raw_begin + block_size); - if (begin >= end) { return {}; } - return padded_string_view(data.data() + begin, end - begin, - data.capacity() - begin); -} - -} // namespace simdjson - - -#endif // SIMDJSON_PADDED_STRING_VIEW_INL_H -/* end file simdjson/padded_string_view-inl.h */ - -#include -#include - -#if SIMDJSON_HAS_UNISTD_H -#include -#include -#include -#include -#include -#endif -// On Windows, `padded_memory_map` (when it is enabled) depends on types and -// functions declared in . We deliberately do NOT include that -// header here: users of simdjson who want `padded_memory_map` on Windows -// must include themselves *before* including this header. See -// padded_string.h for the detection logic. - -namespace simdjson { -namespace internal { - -// The allocate_padded_buffer function is a low-level function to allocate memory -// with padding so we can read past the "length" bytes safely. It is used by -// the padded_string class automatically. It returns nullptr in case -// of error: the caller should check for a null pointer. -// The length parameter is the maximum size in bytes of the string. -// The caller is responsible to free the memory (e.g., delete[] (...)). -inline char *allocate_padded_buffer(size_t length) noexcept { - const size_t totalpaddedlength = length + SIMDJSON_PADDING; - if(totalpaddedlength(1UL<<20)) { - return nullptr; - } -#endif - - char *padded_buffer = new (std::nothrow) char[totalpaddedlength]; - if (padded_buffer == nullptr) { - return nullptr; - } - // We write nulls in the padded region to avoid having uninitialized - // content which may trigger warning for some sanitizers - std::memset(padded_buffer + length, 0, totalpaddedlength - length); - return padded_buffer; -} // allocate_padded_buffer() - -} // namespace internal - - -inline padded_string::padded_string() noexcept = default; -inline padded_string::padded_string(size_t length) noexcept - : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { - if (data_ptr == nullptr) { viable_size = 0; } -} -inline padded_string::padded_string(const char *data, size_t length) noexcept - : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { - if ((data != nullptr) && (data_ptr != nullptr)) { - std::memcpy(data_ptr, data, length); - } - if (data_ptr == nullptr) { - viable_size = 0; - } -} -#ifdef __cpp_char8_t -inline padded_string::padded_string(const char8_t *data, size_t length) noexcept - : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { - if ((data != nullptr) && (data_ptr != nullptr)) { - std::memcpy(data_ptr, reinterpret_cast(data), length); - } - if (data_ptr == nullptr) { - viable_size = 0; - } -} -#endif -// note: do not pass std::string arguments by value -inline padded_string::padded_string(const std::string & str_ ) noexcept - : viable_size(str_.size()), data_ptr(internal::allocate_padded_buffer(str_.size())) { - if (data_ptr == nullptr) { - viable_size = 0; - } else { - std::memcpy(data_ptr, str_.data(), str_.size()); - } -} -// note: do pass std::string_view arguments by value -inline padded_string::padded_string(std::string_view sv_) noexcept - : viable_size(sv_.size()), data_ptr(internal::allocate_padded_buffer(sv_.size())) { - if(simdjson_unlikely(!data_ptr)) { - //allocation failed or zero size - viable_size = 0; - return; - } - if (sv_.size()) { - std::memcpy(data_ptr, sv_.data(), sv_.size()); - } -} -inline padded_string::padded_string(padded_string &&o) noexcept - : viable_size(o.viable_size), data_ptr(o.data_ptr) { - o.data_ptr = nullptr; // we take ownership - o.viable_size = 0; -} - -inline padded_string &padded_string::operator=(padded_string &&o) noexcept { - delete[] data_ptr; - data_ptr = o.data_ptr; - viable_size = o.viable_size; - o.data_ptr = nullptr; // we take ownership - o.viable_size = 0; - return *this; -} - -inline void padded_string::swap(padded_string &o) noexcept { - size_t tmp_viable_size = viable_size; - char *tmp_data_ptr = data_ptr; - viable_size = o.viable_size; - data_ptr = o.data_ptr; - o.data_ptr = tmp_data_ptr; - o.viable_size = tmp_viable_size; -} - -inline padded_string::~padded_string() noexcept { - delete[] data_ptr; -} - -inline size_t padded_string::size() const noexcept { return viable_size; } - -inline size_t padded_string::length() const noexcept { return viable_size; } - -inline const char *padded_string::data() const noexcept { return data_ptr; } - -inline char *padded_string::data() noexcept { return data_ptr; } - -inline bool padded_string::append(const char *data, size_t length) noexcept { - if (length == 0) { - return true; // Nothing to append - } - size_t new_size = viable_size + length; - if (new_size < viable_size) { - // Overflow, cannot append - return false; - } - char *new_data_ptr = internal::allocate_padded_buffer(new_size); - if (new_data_ptr == nullptr) { - // Allocation failed, cannot append - return false; - } - // Copy existing data - if (viable_size > 0) { - std::memcpy(new_data_ptr, data_ptr, viable_size); - } - // Copy new data - std::memcpy(new_data_ptr + viable_size, data, length); - // Update - delete[] data_ptr; - data_ptr = new_data_ptr; - viable_size = new_size; - return true; -} - -inline padded_string::operator std::string_view() const simdjson_lifetime_bound { return std::string_view(data(), length()); } - -inline padded_string::operator padded_string_view() const noexcept simdjson_lifetime_bound { - return padded_string_view(data(), length(), length() + SIMDJSON_PADDING); -} - -inline simdjson_result padded_string::load(std::string_view filename) noexcept { - // std::string_view is not guaranteed to be null-terminated, but std::fopen requires - // a null-terminated C string. Construct a temporary std::string to ensure null-termination. - const std::string null_terminated_filename(filename); - // Open the file - SIMDJSON_PUSH_DISABLE_WARNINGS - SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe - std::FILE *fp = std::fopen(null_terminated_filename.c_str(), "rb"); - SIMDJSON_POP_DISABLE_WARNINGS - - if (fp == nullptr) { - return IO_ERROR; - } - - // Get the file size - int ret; -#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS - ret = _fseeki64(fp, 0, SEEK_END); -#else - ret = std::fseek(fp, 0, SEEK_END); -#endif // _WIN64 - if(ret < 0) { - std::fclose(fp); - return IO_ERROR; - } -#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS - __int64 llen = _ftelli64(fp); - if(llen == -1L) { - std::fclose(fp); - return IO_ERROR; - } -#else - long llen = std::ftell(fp); - if((llen < 0) || (llen == LONG_MAX)) { - std::fclose(fp); - return IO_ERROR; - } -#endif - - // Allocate the padded_string - size_t len = static_cast(llen); - padded_string s(len); - if (s.data() == nullptr) { - std::fclose(fp); - return MEMALLOC; - } - - // Read the padded_string - std::rewind(fp); - size_t bytes_read = std::fread(s.data(), 1, len, fp); - if (std::fclose(fp) != 0 || bytes_read != len) { - return IO_ERROR; - } - - return s; -} - -#if defined(_WIN32) && SIMDJSON_CPLUSPLUS17 -inline simdjson_result padded_string::load(std::wstring_view filename) noexcept { - // std::wstring_view is not guaranteed to be null-terminated, but _wfopen requires - // a null-terminated wide C string. Construct a temporary std::wstring to ensure null-termination. - const std::wstring null_terminated_filename(filename); - // Open the file using the wide characters - SIMDJSON_PUSH_DISABLE_WARNINGS - SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe - std::FILE *fp = _wfopen(null_terminated_filename.c_str(), L"rb"); - SIMDJSON_POP_DISABLE_WARNINGS - - if (fp == nullptr) { - return IO_ERROR; - } - - // Get the file size - int ret; -#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS - ret = _fseeki64(fp, 0, SEEK_END); -#else - ret = std::fseek(fp, 0, SEEK_END); -#endif // _WIN64 - if(ret < 0) { - std::fclose(fp); - return IO_ERROR; - } -#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS - __int64 llen = _ftelli64(fp); - if(llen == -1L) { - std::fclose(fp); - return IO_ERROR; - } -#else - long llen = std::ftell(fp); - if((llen < 0) || (llen == LONG_MAX)) { - std::fclose(fp); - return IO_ERROR; - } -#endif - - // Allocate the padded_string - size_t len = static_cast(llen); - padded_string s(len); - if (s.data() == nullptr) { - std::fclose(fp); - return MEMALLOC; - } - - // Read the padded_string - std::rewind(fp); - size_t bytes_read = std::fread(s.data(), 1, len, fp); - if (std::fclose(fp) != 0 || bytes_read != len) { - return IO_ERROR; - } - - return s; -} -#endif - -// padded_string_builder implementations - -inline padded_string_builder::padded_string_builder() noexcept = default; - -inline padded_string_builder::padded_string_builder(size_t new_capacity) noexcept { - if (new_capacity > 0) { - data = internal::allocate_padded_buffer(new_capacity); - if (data != nullptr) { - this->capacity = new_capacity; - } - } -} - -inline padded_string_builder::padded_string_builder(padded_string_builder &&o) noexcept - : size(o.size), capacity(o.capacity), data(o.data) { - o.size = 0; - o.capacity = 0; - o.data = nullptr; -} - -inline padded_string_builder &padded_string_builder::operator=(padded_string_builder &&o) noexcept { - if (this != &o) { - delete[] data; - size = o.size; - capacity = o.capacity; - data = o.data; - o.size = 0; - o.capacity = 0; - o.data = nullptr; - } - return *this; -} - -inline padded_string_builder::~padded_string_builder() noexcept { - delete[] data; -} - -inline bool padded_string_builder::append(const char *newdata, size_t length) noexcept { - if (length == 0) { - return true; - } - if (!reserve(length)) { - return false; - } - std::memcpy(data + size, newdata, length); - size += length; - return true; -} - -inline bool padded_string_builder::append(std::string_view sv) noexcept { - return append(sv.data(), sv.size()); -} - -inline size_t padded_string_builder::length() const noexcept { - return size; -} - -inline padded_string padded_string_builder::build() const noexcept { - return padded_string(data, size); -} - -inline padded_string padded_string_builder::convert() noexcept { - padded_string result{}; - result.data_ptr = data; - result.viable_size = size; - data = nullptr; - size = 0; - capacity = 0; - return result; -} - -inline bool padded_string_builder::reserve(size_t additional) noexcept { - if (simdjson_unlikely(additional + size < size)) { - return false; // overflow: cannot satisfy request - } - size_t needed = size + additional; - if (needed <= capacity) { - return true; - } - size_t new_capacity = needed; - // We are going to grow the capacity exponentially to avoid - // repeated allocations. - if (new_capacity < 4096) { - new_capacity *= 2; - // overflow guard: ensure new_capacity + new_capacity/2 does not overflow - } else if (new_capacity + new_capacity / 2 > new_capacity) { - new_capacity += new_capacity / 2; // grow by 1.5x - } - char *new_data = internal::allocate_padded_buffer(new_capacity); - if (new_data == nullptr) { - return false; // Allocation failed - } - if (size > 0) { - std::memcpy(new_data, data, size); - } - delete[] data; - data = new_data; - capacity = new_capacity; - return true; -} - - -#if SIMDJSON_HAS_PADDED_MEMORY_MAP - -#if SIMDJSON_HAS_UNISTD_H -simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept { - - int fd = open(filename, O_RDONLY); - if (fd == -1) { - return; // file not found or cannot be opened, data will be nullptr - } - struct stat st; - if (fstat(fd, &st) == -1) { - close(fd); - return; // failed to get file size, data will be nullptr - } - size = static_cast(st.st_size); - size_t total_size = size + simdjson::SIMDJSON_PADDING; - void *anon_map = - mmap(NULL, total_size, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (anon_map == MAP_FAILED) { - close(fd); - return; // failed to create anonymous mapping, data will be nullptr - } - void *file_map = - mmap(anon_map, size, PROT_READ, MAP_SHARED | MAP_FIXED, fd, 0); - if (file_map == MAP_FAILED) { - munmap(anon_map, total_size); - close(fd); - return; // failed to mmap file, data will be nullptr - } - data = static_cast(file_map); - close(fd); // no longer needed after mapping -} - -simdjson_inline padded_memory_map::~padded_memory_map() noexcept { - if (data != nullptr) { - munmap(const_cast(data), size + simdjson::SIMDJSON_PADDING); - } -} -#elif defined(_WIN32) -// Windows zero-copy implementation using placeholder virtual memory. -// -// We use the modern Windows memory APIs (VirtualAlloc2, CreateFileMapping2, -// MapViewOfFile3 -- available since Windows 10 1803) to map the file into a -// contiguous virtual address range that includes at least SIMDJSON_PADDING -// zero bytes after the file content, with no data copies. -// -// Strategy: -// 1. If rounding the file size up to the allocation granularity already -// exceeds file_size + SIMDJSON_PADDING, the OS page zero-fill provides -// the padding and we use a simple MapViewOfFile3 call. -// 2. Otherwise we reserve a contiguous placeholder region via VirtualAlloc2, -// split it at the granularity-aligned file boundary, map the file into -// the first part, and commit zero pages for the second part (padding). -simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept { - HANDLE file_handle = ::CreateFileA( - filename, GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) { - return; - } - LARGE_INTEGER file_size_li; - if (!::GetFileSizeEx(file_handle, &file_size_li) || file_size_li.QuadPart < 0) { - ::CloseHandle(file_handle); - return; - } -#if SIMDJSON_IS_32BITS - if (static_cast(file_size_li.QuadPart) > - static_cast(SIZE_MAX - simdjson::SIMDJSON_PADDING)) { - ::CloseHandle(file_handle); - return; - } -#endif - size = static_cast(file_size_li.QuadPart); - if (size == 0) { - ::CloseHandle(file_handle); - return; - } - - HANDLE section = ::CreateFileMapping2( - file_handle, NULL, FILE_MAP_READ, PAGE_READONLY, - 0, 0, NULL, NULL, 0); - ::CloseHandle(file_handle); - if (section == NULL) { - return; - } - - SYSTEM_INFO si; - ::GetSystemInfo(&si); - const size_t granularity = static_cast(si.dwAllocationGranularity); - const size_t file_region = (size + granularity - 1) & ~(granularity - 1); - const size_t total_needed = size + simdjson::SIMDJSON_PADDING; - - if (file_region >= total_needed) { - // The zero-fill in the last page already covers the padding. - PVOID view = ::MapViewOfFile3( - section, ::GetCurrentProcess(), NULL, 0, 0, - 0, PAGE_READONLY, NULL, 0); - ::CloseHandle(section); - if (view != NULL) { - data = static_cast(view); - } - return; - } - - // We need extra zero pages beyond the file region. Use the placeholder API - // to get a contiguous virtual address range spanning both the file mapping - // and the zero-filled padding. - const size_t padding_region = - ((total_needed - file_region) + granularity - 1) & ~(granularity - 1); - const size_t reserve_size = file_region + padding_region; - - // Reserve a contiguous placeholder. - PVOID placeholder = ::VirtualAlloc2( - ::GetCurrentProcess(), NULL, reserve_size, - MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, NULL, 0); - if (placeholder == NULL) { - ::CloseHandle(section); - return; - } - - // Split into two placeholders at the file_region boundary. - if (!::VirtualFree(placeholder, file_region, - MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)) { - ::VirtualFree(placeholder, 0, MEM_RELEASE); - ::CloseHandle(section); - return; - } - - // Map the file into the first placeholder. - PVOID file_view = ::MapViewOfFile3( - section, ::GetCurrentProcess(), placeholder, 0, file_region, - MEM_REPLACE_PLACEHOLDER, PAGE_READONLY, NULL, 0); - ::CloseHandle(section); - if (file_view == NULL) { - ::VirtualFree(placeholder, 0, MEM_RELEASE); - ::VirtualFree(static_cast(placeholder) + file_region, - 0, MEM_RELEASE); - return; - } - - // Commit zero pages in the second placeholder (the padding). - void *pad = static_cast(placeholder) + file_region; - PVOID padding_ptr = ::VirtualAlloc2( - ::GetCurrentProcess(), pad, padding_region, - MEM_REPLACE_PLACEHOLDER | MEM_COMMIT, PAGE_READONLY, NULL, 0); - if (padding_ptr == NULL) { - ::UnmapViewOfFile(file_view); - ::VirtualFree(pad, 0, MEM_RELEASE); - return; - } - - data = static_cast(file_view); - padding_view_ = padding_ptr; -} - -simdjson_inline padded_memory_map::~padded_memory_map() noexcept { - if (data == nullptr) { return; } - ::UnmapViewOfFile(data); - if (padding_view_ != nullptr) { - ::VirtualFree(padding_view_, 0, MEM_RELEASE); - } -} -#endif // POSIX or _WIN32 - -simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noexcept simdjson_lifetime_bound { - if(!is_valid()) { - return simdjson::padded_string_view(); // return an empty view if mapping failed - } - return simdjson::padded_string_view(data, size, size + simdjson::SIMDJSON_PADDING); -} - -simdjson_inline bool padded_memory_map::is_valid() const noexcept { - return data != nullptr; -} - -#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP - -} // namespace simdjson - -inline simdjson::padded_string operator ""_padded(const char *str, size_t len) { - return simdjson::padded_string(str, len); -} -#ifdef __cpp_char8_t -inline simdjson::padded_string operator ""_padded(const char8_t *str, size_t len) { - return simdjson::padded_string(reinterpret_cast(str), len); -} -#endif - -#endif // SIMDJSON_PADDED_STRING_INL_H -/* end file simdjson/padded_string-inl.h */ -/* skipped duplicate #include "simdjson/padded_string_view.h" */ -/* skipped duplicate #include "simdjson/padded_string_view-inl.h" */ - -/* including simdjson/dom.h: #include "simdjson/dom.h" */ -/* begin file simdjson/dom.h */ -#ifndef SIMDJSON_DOM_H -#define SIMDJSON_DOM_H - -/* including simdjson/dom/base.h: #include "simdjson/dom/base.h" */ -/* begin file simdjson/dom/base.h */ -#ifndef SIMDJSON_DOM_BASE_H -#define SIMDJSON_DOM_BASE_H - -/* skipped duplicate #include "simdjson/base.h" */ - -namespace simdjson { - -/** - * @brief A DOM API on top of the simdjson parser. - */ -namespace dom { - -/** The default batch size for parser.parse_many() and parser.load_many() */ -static constexpr size_t DEFAULT_BATCH_SIZE = 1000000; -/** - * Some adversary might try to set the batch size to 0 or 1, which might cause problems. - * We set a minimum of 32B since anything else is highly likely to be an error. In practice, - * most users will want a much larger batch size. - * - * All non-negative MINIMAL_BATCH_SIZE values should be 'safe' except that, obviously, no JSON - * document can ever span 0 or 1 byte and that very large values would create memory allocation issues. - */ -static constexpr size_t MINIMAL_BATCH_SIZE = 32; - -/** - * It is wasteful to allocate memory for tiny documents (e.g., 4 bytes). - */ -static constexpr size_t MINIMAL_DOCUMENT_CAPACITY = 32; - -class array; -class document; -class document_stream; -class element; -class key_value_pair; -class object; -class parser; - -#ifdef SIMDJSON_THREADS_ENABLED -struct stage1_worker; -#endif // SIMDJSON_THREADS_ENABLED - -} // namespace dom - -namespace internal { - -template -class string_builder; -class tape_ref; - -} // namespace internal - -} // namespace simdjson - -#endif // SIMDJSON_DOM_BASE_H -/* end file simdjson/dom/base.h */ -/* including simdjson/dom/array.h: #include "simdjson/dom/array.h" */ -/* begin file simdjson/dom/array.h */ -#ifndef SIMDJSON_DOM_ARRAY_H -#define SIMDJSON_DOM_ARRAY_H - -#include - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* including simdjson/internal/tape_ref.h: #include "simdjson/internal/tape_ref.h" */ -/* begin file simdjson/internal/tape_ref.h */ -#ifndef SIMDJSON_INTERNAL_TAPE_REF_H -#define SIMDJSON_INTERNAL_TAPE_REF_H - -/* skipped duplicate #include "simdjson/base.h" */ - -namespace simdjson { -namespace dom { -class document; -} // namespace dom - -namespace internal { - -/** - * A reference to an element on the tape. Internal only. - */ -class tape_ref { -public: - simdjson_inline tape_ref() noexcept; - simdjson_inline tape_ref(const dom::document *doc, size_t json_index) noexcept; - inline size_t after_element() const noexcept; - // The reference must point to an element boundary inside the array whose - // opening tag is at array_start, or to that array's closing tag. - inline size_t before_element(size_t array_start) const noexcept; - simdjson_inline tape_type tape_ref_type() const noexcept; - simdjson_inline uint64_t tape_value() const noexcept; - simdjson_inline bool is_double() const noexcept; - simdjson_inline bool is_int64() const noexcept; - simdjson_inline bool is_uint64() const noexcept; - simdjson_inline bool is_false() const noexcept; - simdjson_inline bool is_true() const noexcept; - simdjson_inline bool is_null_on_tape() const noexcept;// different name to avoid clash with is_null. - simdjson_inline uint32_t matching_brace_index() const noexcept; - simdjson_inline uint32_t scope_count() const noexcept; - template - simdjson_inline T next_tape_value() const noexcept; - simdjson_inline uint32_t get_string_length() const noexcept; - simdjson_inline const char * get_c_str() const noexcept; - inline std::string_view get_string_view() const noexcept; - simdjson_inline bool is_document_root() const noexcept; - simdjson_inline bool usable() const noexcept; - - /** The document this element references. */ - const dom::document *doc; - - /** The index of this element on `doc.tape[]` */ - size_t json_index; -}; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_TAPE_REF_H -/* end file simdjson/internal/tape_ref.h */ - -namespace simdjson { -namespace dom { - -/** - * JSON array. - */ -class array { -public: - /** Create a new, invalid array */ - simdjson_inline array() noexcept; - - class iterator { - public: - using value_type = element; - using difference_type = std::ptrdiff_t; - using pointer = void; - using reference = value_type; - using iterator_category = std::forward_iterator_tag; - - /** - * Get the actual value - */ - inline reference operator*() const noexcept; - /** - * Get the next value. - * - * Part of the std::iterator interface. - */ - inline iterator& operator++() noexcept; - /** - * Get the next value. - * - * Part of the std::iterator interface. - */ - inline iterator operator++(int) noexcept; - /** - * Check if these values come from the same place in the JSON. - * - * Part of the std::iterator interface. - */ - inline bool operator!=(const iterator& other) const noexcept; - inline bool operator==(const iterator& other) const noexcept; - - inline bool operator<(const iterator& other) const noexcept; - inline bool operator<=(const iterator& other) const noexcept; - inline bool operator>=(const iterator& other) const noexcept; - inline bool operator>(const iterator& other) const noexcept; - - iterator() noexcept = default; - iterator(const iterator&) noexcept = default; - iterator& operator=(const iterator&) noexcept = default; - private: - simdjson_inline iterator(const internal::tape_ref &tape) noexcept; - internal::tape_ref tape{}; - friend class array; - }; - - /** - * A forward iterator that visits the array's elements in reverse order. - * Like iterator, it returns element handles by value and does not own the - * document. No allocation or modification of the document is performed. - */ - class reverse_iterator { - public: - using value_type = element; - using difference_type = std::ptrdiff_t; - using pointer = void; - using reference = value_type; - using iterator_category = std::forward_iterator_tag; - - inline reference operator*() const noexcept; - inline reverse_iterator& operator++() noexcept; - inline reverse_iterator operator++(int) noexcept; - inline bool operator==(const reverse_iterator& other) const noexcept; - inline bool operator!=(const reverse_iterator& other) const noexcept; - - reverse_iterator() noexcept = default; - reverse_iterator(const reverse_iterator&) noexcept = default; - reverse_iterator& operator=(const reverse_iterator&) noexcept = default; - private: - simdjson_inline reverse_iterator(const internal::tape_ref &tape, size_t array_start) noexcept; - internal::tape_ref tape{}; - size_t array_start{}; - friend class array; - }; - - /** - * Return the last array element, or rend() for an empty array. - * Incrementing the returned iterator moves toward the first element. - * A complete traversal takes O(n) time and O(1) additional space, where n - * is the number of immediate elements in the array. - * Finding the last or previous element can take O(n) time in the worst case when - * numeric payloads equal numeric type markers. Increments are amortized O(1) - * over a complete traversal; nested values do not increase this bound. - */ - inline reverse_iterator rbegin() const noexcept; - /** Return the reverse traversal sentinel, before the first element. */ - inline reverse_iterator rend() const noexcept; - - /** - * Return the first array element. - * - * Part of the std::iterable interface. - */ - inline iterator begin() const noexcept; - /** - * One past the last array element. - * - * Part of the std::iterable interface. - */ - inline iterator end() const noexcept; - /** - * Get the size of the array (number of immediate children). - * It is a saturated value with a maximum of 0xFFFFFF: if the value - * is 0xFFFFFF then the size is 0xFFFFFF or greater. - */ - inline size_t size() const noexcept; - /** - * Get the total number of slots used by this array on the tape. - * - * Note that this is not the same thing as `size()`, which reports the - * number of actual elements within an array (not counting its children). - * - * Since an element can use 1 or 2 slots on the tape, you can only use this - * to figure out the total size of an array (including its children, - * recursively) if you know its structure ahead of time. - **/ - inline size_t number_of_slots() const noexcept; - /** - * Get the value associated with the given JSON pointer. We use the RFC 6901 - * https://tools.ietf.org/html/rfc6901 standard, interpreting the current node - * as the root of its own JSON document. - * - * dom::parser parser; - * array a = parser.parse(R"([ { "foo": { "a": [ 10, 20, 30 ] }} ])"_padded); - * a.at_pointer("/0/foo/a/1") == 20 - * a.at_pointer("0")["foo"]["a"].at(1) == 20 - * - * @return The value associated with the given JSON pointer, or: - * - NO_SUCH_FIELD if a field does not exist in an object - * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length - * - INCORRECT_TYPE if a non-integer is used to access an array - * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed - */ - inline simdjson_result at_pointer(std::string_view json_pointer) const noexcept; - - /** - * Recursive function which processes the JSON path of each child element - */ - inline void process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept; - - - /** - * Adds support for JSONPath expression with wildcards '*' - */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) const noexcept; - - /** - * Get the value associated with the given JSONPath expression. We only support - * JSONPath queries that trivially convertible to JSON Pointer queries: key - * names and array indices. - * - * https://www.rfc-editor.org/rfc/rfc9535 (RFC 9535) - * - * @return The value associated with the given JSONPath expression, or: - * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails - * - NO_SUCH_FIELD if a field does not exist in an object - * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length - * - INCORRECT_TYPE if a non-integer is used to access an array - */ - inline simdjson_result at_path(std::string_view json_path) const noexcept; - - /** - * Get the value at the given index. This function has linear-time complexity and - * is equivalent to the following: - * - * size_t i=0; - * for (auto element : *this) { - * if (i == index) { return element; } - * i++; - * } - * return INDEX_OUT_OF_BOUNDS; - * - * Avoid calling the at() function repeatedly. - * - * @return The value at the given index, or: - * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length - */ - inline simdjson_result at(size_t index) const noexcept; - - /** - * Gets the values of items in an array element - * This function has linear-time complexity: the values are checked one by one. - * - * @return The child elements of an array - */ - - inline std::vector& get_values(std::vector& out) const noexcept; - - /** - * Implicitly convert object to element - */ - inline operator element() const noexcept; - -private: - simdjson_inline array(const internal::tape_ref &tape) noexcept; - internal::tape_ref tape{}; - friend class element; - friend struct simdjson_result; - template - friend class simdjson::internal::string_builder; -}; - - -} // namespace dom - -/** The result of a JSON conversion that may fail. */ -template<> -struct simdjson_result : public internal::simdjson_result_base { -public: - simdjson_inline simdjson_result() noexcept; ///< @private - simdjson_inline simdjson_result(dom::array value) noexcept; ///< @private - simdjson_inline simdjson_result(error_code error) noexcept; ///< @private - - inline simdjson_result at_pointer(std::string_view json_pointer) const noexcept; - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) const noexcept; - inline simdjson_result at_path(std::string_view json_path) const noexcept; - inline simdjson_result at(size_t index) const noexcept; - inline std::vector& get_values(std::vector& out) const noexcept; - -#if SIMDJSON_EXCEPTIONS - inline dom::array::iterator begin() const noexcept(false); - inline dom::array::iterator end() const noexcept(false); - inline dom::array::reverse_iterator rbegin() const noexcept(false); - inline dom::array::reverse_iterator rend() const noexcept(false); - inline size_t size() const noexcept(false); -#endif // SIMDJSON_EXCEPTIONS -}; - - - -} // namespace simdjson - -#if SIMDJSON_SUPPORTS_RANGES -namespace std { -namespace ranges { -template<> -inline constexpr bool enable_view = true; -#if SIMDJSON_EXCEPTIONS -template<> -inline constexpr bool enable_view> = true; -#endif // SIMDJSON_EXCEPTIONS -} // namespace ranges -} // namespace std -#endif // SIMDJSON_SUPPORTS_RANGES - -#endif // SIMDJSON_DOM_ARRAY_H -/* end file simdjson/dom/array.h */ -/* including simdjson/dom/document_stream.h: #include "simdjson/dom/document_stream.h" */ -/* begin file simdjson/dom/document_stream.h */ -#ifndef SIMDJSON_DOCUMENT_STREAM_H -#define SIMDJSON_DOCUMENT_STREAM_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* including simdjson/dom/parser.h: #include "simdjson/dom/parser.h" */ -/* begin file simdjson/dom/parser.h */ -#ifndef SIMDJSON_DOM_PARSER_H -#define SIMDJSON_DOM_PARSER_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* including simdjson/dom/document.h: #include "simdjson/dom/document.h" */ -/* begin file simdjson/dom/document.h */ -#ifndef SIMDJSON_DOM_DOCUMENT_H -#define SIMDJSON_DOM_DOCUMENT_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ - -#include - -namespace simdjson { -namespace dom { - -/** - * A parsed JSON document. - * - * This class cannot be copied, only moved, to avoid unintended allocations. - */ -class document { -public: - /** - * Create a document container with zero capacity. - * - * The parser will allocate capacity as needed. - */ - document() noexcept = default; - ~document() noexcept = default; - - /** - * Take another document's buffers. - * - * @param other The document to take. Its capacity is zeroed and it is invalidated. - */ - document(document &&other) noexcept = default; - /** @private */ - document(const document &) = delete; // Disallow copying - /** - * Take another document's buffers. - * - * @param other The document to take. Its capacity is zeroed. - */ - document &operator=(document &&other) noexcept = default; - /** @private */ - document &operator=(const document &) = delete; // Disallow copying - - /** - * Get the root element of this document as a JSON array. - */ - element root() const noexcept; - - /** - * @private Dump the raw tape for debugging. - * - * @param os the stream to output to. - * @return false if the tape is likely wrong (e.g., you did not parse a valid JSON). - */ - bool dump_raw_tape(std::ostream &os) const noexcept; - - /** @private Structural values. */ - std::unique_ptr tape{}; - - /** @private String values. - * - * Should be at least byte_capacity. - */ - std::unique_ptr string_buf{}; - /** @private Allocate memory to support - * input JSON documents of up to len bytes. - * - * When calling this function, you lose - * all the data. - * - * The memory allocation is strict: you - * can you use this function to increase - * or lower the amount of allocated memory. - * Passing zero clears the memory. - */ - error_code allocate(size_t len) noexcept; - /** @private Capacity in bytes, in terms - * of how many bytes of input JSON we can - * support. - */ - size_t capacity() const noexcept; - - -private: - size_t allocated_capacity{0}; - friend class parser; -}; // class document - -} // namespace dom -} // namespace simdjson - -#endif // SIMDJSON_DOM_DOCUMENT_H -/* end file simdjson/dom/document.h */ - -namespace simdjson { - -namespace dom { - -/** - * A persistent document parser. - * - * The parser is designed to be reused, holding the internal buffers necessary to do parsing, - * as well as memory for a single document. The parsed document is overwritten on each parse. - * - * This class cannot be copied, only moved, to avoid unintended allocations. - * - * @note Moving a parser instance may invalidate "dom::element" instances. If you need to - * preserve both the "dom::element" instances and the parser, consider wrapping the parser - * instance in a std::unique_ptr instance: - * - * std::unique_ptr parser(new dom::parser{}); - * auto error = parser->load(f).get(root); - * - * You can then move std::unique_ptr safely. - * - * @note This is not thread safe: one parser cannot produce two documents at the same time! - */ -class parser { -public: - /** - * Create a JSON parser. - * - * The new parser will have zero capacity. - * - * @param max_capacity The maximum document length the parser can automatically handle. The parser - * will allocate more capacity on an as needed basis (when it sees documents too big to handle) - * up to this amount. The parser still starts with zero capacity no matter what this number is: - * to allocate an initial capacity, call allocate() after constructing the parser. - * Defaults to SIMDJSON_MAXSIZE_BYTES (the largest single document simdjson can process). - */ - simdjson_inline explicit parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept; - /** - * Take another parser's buffers and state. - * - * @param other The parser to take. Its capacity is zeroed. - */ - simdjson_inline parser(parser &&other) noexcept; - parser(const parser &) = delete; ///< @private Disallow copying - /** - * Take another parser's buffers and state. - * - * @param other The parser to take. Its capacity is zeroed. - */ - simdjson_inline parser &operator=(parser &&other) noexcept; - parser &operator=(const parser &) = delete; ///< @private Disallow copying - - /** Deallocate the JSON parser. */ - ~parser()=default; - - /** - * Load a JSON document from a file and return a reference to it. - * - * dom::parser parser; - * const element doc = parser.load("jsonexamples/twitter.json"); - * - * The function is eager: the file's content is loaded in memory inside the parser instance - * and immediately parsed. The file can be deleted after the `parser.load` call. - * - * ### IMPORTANT: Document Lifetime - * - * The JSON document still lives in the parser: this is the most efficient way to parse JSON - * documents because it reuses the same buffers, but you *must* use the document before you - * destroy the parser or call parse() again. - * - * Moving the parser instance is safe, but it invalidates the element instances. You may store - * the parser instance without moving it by wrapping it inside an `unique_ptr` instance like - * so: `std::unique_ptr parser(new dom::parser{});`. - * - * ### Parser Capacity - * - * If the parser's current capacity is less than the file length, it will allocate enough capacity - * to handle it (up to max_capacity). - * - * ## Windows and Unicode - * - * Windows users who need to read files with non-ANSI characters in the - * name should set their code page to UTF-8 (65001) before calling this - * function. This should be the default with Windows 11 and better. - * Further, they may use the AreFileApisANSI function to determine whether - * the filename is interpreted using the ANSI or the system default OEM - * codepage, and they may call SetFileApisToOEM accordingly. - * - * @param path The path to load. - * @return The document, or an error: - * - IO_ERROR if there was an error opening or reading the file. - * Be mindful that on some 32-bit systems, - * the file size might be limited to 2 GB. - * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. - * - CAPACITY if the parser does not have enough capacity and len > max_capacity. - * - other json errors if parsing fails. You should not rely on these errors to always the same for the - * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). - */ - inline simdjson_result load(std::string_view path) & noexcept; - inline simdjson_result load(std::string_view path) && = delete ; - - /** - * Load a JSON document from a file into a provide document instance and return a temporary reference to it. - * It is similar to the function `load` except that instead of parsing into the internal - * `document` instance associated with the parser, it allows the user to provide a document - * instance. - * - * dom::parser parser; - * dom::document doc; - * element doc_root = parser.load_into_document(doc, "jsonexamples/twitter.json"); - * - * The function is eager: the file's content is loaded in memory inside the parser instance - * and immediately parsed. The file can be deleted after the `parser.load_into_document` call. - * - * ### IMPORTANT: Document Lifetime - * - * After the call to load_into_document, the parser is no longer needed. - * - * The JSON document lives in the document instance: you must keep the document - * instance alive while you navigate through it (i.e., used the returned value from - * load_into_document). You are encourage to reuse the document instance - * many times with new data to avoid reallocations: - * - * dom::document doc; - * element doc_root1 = parser.load_into_document(doc, "jsonexamples/twitter.json"); - * //... doc_root1 is a pointer inside doc - * element doc_root2 = parser.load_into_document(doc, "jsonexamples/twitter.json"); - * //... doc_root2 is a pointer inside doc - * // at this point doc_root1 is no longer safe - * - * Moving the document instance is safe, but it invalidates the element instances. After - * moving a document, you can recover safe access to the document root with its `root()` method. - * - * @param doc The document instance where the parsed data will be stored (on success). - * @param path The path to load. - * @return The document, or an error: - * - IO_ERROR if there was an error opening or reading the file. - * Be mindful that on some 32-bit systems, - * the file size might be limited to 2 GB. - * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. - * - CAPACITY if the parser does not have enough capacity and len > max_capacity. - * - other json errors if parsing fails. You should not rely on these errors to always the same for the - * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). - */ - inline simdjson_result load_into_document(document& doc, std::string_view path) & noexcept; - inline simdjson_result load_into_document(document& doc, std::string_view path) && =delete; - - /** - * Parse a JSON document and return a temporary reference to it. - * - * dom::parser parser; - * element doc_root = parser.parse(buf, len); - * - * The function eagerly parses the input: the input can be modified and discarded after - * the `parser.parse(buf, len)` call has completed. - * - * ### IMPORTANT: Document Lifetime - * - * The JSON document still lives in the parser: this is the most efficient way to parse JSON - * documents because it reuses the same buffers, but you *must* use the document before you - * destroy the parser or call parse() again. - * - * Moving the parser instance is safe, but it invalidates the element instances. You may store - * the parser instance without moving it by wrapping it inside an `unique_ptr` instance like - * so: `std::unique_ptr parser(new dom::parser{});`. - * - * ### REQUIRED: Buffer Padding - * - * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what - * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you - * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the - * SIMDJSON_PADDING bytes to avoid runtime warnings. - * - * If realloc_if_needed is true (the default), it is assumed that the buffer does *not* have enough padding, - * and it is copied into an enlarged temporary buffer before parsing. Thus the following is safe: - * - * const char *json = R"({"key":"value"})"; - * const size_t json_len = std::strlen(json); - * simdjson::dom::parser parser; - * simdjson::dom::element element = parser.parse(json, json_len); - * - * If you set realloc_if_needed to false (e.g., parser.parse(json, json_len, false)), - * you must provide a buffer with at least SIMDJSON_PADDING extra bytes at the end. - * The benefit of setting realloc_if_needed to false is that you avoid a temporary - * memory allocation and a copy. - * - * The padded bytes may be read. It is not important how you initialize - * these bytes though we recommend a sensible default like null character values or spaces. - * For example, the following low-level code is safe: - * - * const char *json = R"({"key":"value"})"; - * const size_t json_len = std::strlen(json); - * std::unique_ptr padded_json_copy{new char[json_len + SIMDJSON_PADDING]}; - * std::memcpy(padded_json_copy.get(), json, json_len); - * std::memset(padded_json_copy.get() + json_len, '\0', SIMDJSON_PADDING); - * simdjson::dom::parser parser; - * simdjson::dom::element element = parser.parse(padded_json_copy.get(), json_len, false); - * - * ### std::string references - * - * Whenever you pass an std::string reference, the parser may access the bytes beyond the end of - * the string but before the end of the allocated memory (std::string::capacity()). - * If you are using a sanitizer that checks for reading uninitialized bytes or std::string's - * container-overflow checks, you may encounter sanitizer warnings. - * You can safely ignore these warnings. Or you can call simdjson::pad(std::string&) to pad the - * string with SIMDJSON_PADDING spaces: this function returns a simdjson::padding_string_view - * which can be be passed to the parser's parse function: - * - * std::string json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )"; - * element doc = parser.parse(simdjson::pad(json)); - * - * ### Parser Capacity - * - * If the parser's current capacity is less than len, it will allocate enough capacity - * to handle it (up to max_capacity). - * - * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless - * realloc_if_needed is true. - * @param len The length of the JSON. - * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. - * @return An element pointing at the root of the document, or an error: - * - MEMALLOC if realloc_if_needed is true or the parser does not have enough capacity, - * and memory allocation fails. - * - CAPACITY if the parser does not have enough capacity and len > max_capacity. - * - other json errors if parsing fails. You should not rely on these errors to always the same for the - * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). - */ - inline simdjson_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) & noexcept; - inline simdjson_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) && =delete; - /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_inline simdjson_result parse(const char *buf, size_t len, bool realloc_if_needed = true) & noexcept; - simdjson_inline simdjson_result parse(const char *buf, size_t len, bool realloc_if_needed = true) && =delete; - /** @overload parse(const std::string &) */ - simdjson_inline simdjson_result parse(const std::string &s) & noexcept; - simdjson_inline simdjson_result parse(const std::string &s) && =delete; - /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_inline simdjson_result parse(const padded_string &s) & noexcept; - simdjson_inline simdjson_result parse(const padded_string &s) && =delete; - /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_inline simdjson_result parse(const padded_string_view &v) & noexcept; - simdjson_inline simdjson_result parse(const padded_string_view &v) && =delete; - - /** @private We do not want to allow implicit conversion from C string to std::string. */ - simdjson_inline simdjson_result parse(const char *buf) noexcept = delete; - - /** - * Parse a JSON document whose buffer is **not** padded, in place and without - * copying it. - * - * *This feature is currently experimental.* - * - * The standard parse() methods require the input buffer to have at least - * SIMDJSON_PADDING extra readable bytes after the document (or they copy it - * into a padded buffer when realloc_if_needed is true). parse_unpadded() lifts - * that requirement: it parses directly from your buffer of exactly `len` bytes, - * never reading past `buf + len`, and never allocating a full padded copy. - * - * dom::parser parser; - * std::string_view json = get_json(); // no trailing padding needed - * dom::element doc = parser.parse_unpadded(json); - * - * This is the convenient way to use simdjson when you cannot (or do not want - * to) pad your input, e.g. a std::string_view into a larger buffer or a memory - * mapped file whose tail you do not control. It is generally a little slower - * than parsing a padded buffer with parse() (the very end of the document is - * handled with extra care), but it avoids the O(n) copy that - * parse(buf, len, true) performs when realloc_if_needed is true. - * - * The input is read but not modified, and it must remain valid (and the parser - * alive) for as long as you navigate the returned document, exactly like - * parse(buf, len, false). - * - * @param buf The JSON to parse. Only `len` bytes are read; no padding required. - * @param len The length of the JSON. - * @return An element pointing at the root of the document, or an error: - * - MEMALLOC if the parser does not have enough capacity and allocation fails. - * - CAPACITY if the parser does not have enough capacity and len > max_capacity. - * - other json errors if parsing fails. - */ - inline simdjson_result parse_unpadded(const uint8_t *buf, size_t len) & noexcept; - inline simdjson_result parse_unpadded(const uint8_t *buf, size_t len) && =delete; - /** @overload parse_unpadded(const uint8_t *buf, size_t len) */ - simdjson_inline simdjson_result parse_unpadded(const char *buf, size_t len) & noexcept; - simdjson_inline simdjson_result parse_unpadded(const char *buf, size_t len) && =delete; - /** @overload parse_unpadded(const uint8_t *buf, size_t len) */ - simdjson_inline simdjson_result parse_unpadded(std::string_view s) & noexcept; - simdjson_inline simdjson_result parse_unpadded(std::string_view s) && =delete; - - /** - * Parse a non-padded JSON document into a caller-provided document instance, in - * place and without copying. This is to parse_unpadded() what - * parse_into_document() is to parse(). See parse_unpadded() for the padding and - * lifetime semantics. - * - * *This feature is currently experimental.* - * - * @param doc The document instance where the parsed data will be stored (on success). - * @param buf The JSON to parse. Only `len` bytes are read; no padding required. - * @param len The length of the JSON. - */ - inline simdjson_result parse_into_document_unpadded(document& doc, const uint8_t *buf, size_t len) & noexcept; - inline simdjson_result parse_into_document_unpadded(document& doc, const uint8_t *buf, size_t len) && =delete; - - /** - * Parse a JSON document into a provide document instance and return a temporary reference to it. - * It is similar to the function `parse` except that instead of parsing into the internal - * `document` instance associated with the parser, it allows the user to provide a document - * instance. - * - * dom::parser parser; - * dom::document doc; - * element doc_root = parser.parse_into_document(doc, buf, len); - * - * The function eagerly parses the input: the input can be modified and discarded after - * the `parser.parse(buf, len)` call has completed. - * - * ### IMPORTANT: Document Lifetime - * - * After the call to parse_into_document, the parser is no longer needed. - * - * The JSON document lives in the document instance: you must keep the document - * instance alive while you navigate through it (i.e., used the returned value from - * parse_into_document). You are encourage to reuse the document instance - * many times with new data to avoid reallocations: - * - * dom::document doc; - * element doc_root1 = parser.parse_into_document(doc, buf1, len); - * //... doc_root1 is a pointer inside doc - * element doc_root2 = parser.parse_into_document(doc, buf1, len); - * //... doc_root2 is a pointer inside doc - * // at this point doc_root1 is no longer safe - * - * Moving the document instance is safe, but it invalidates the element instances. After - * moving a document, you can recover safe access to the document root with its `root()` method. - * - * @param doc The document instance where the parsed data will be stored (on success). - * @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless - * realloc_if_needed is true. - * @param len The length of the JSON. - * @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding. - * @return An element pointing at the root of document, or an error: - * - MEMALLOC if realloc_if_needed is true or the parser does not have enough capacity, - * and memory allocation fails. - * - CAPACITY if the parser does not have enough capacity and len > max_capacity. - * - other json errors if parsing fails. You should not rely on these errors to always the same for the - * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). - */ - inline simdjson_result parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = true) & noexcept; - inline simdjson_result parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = true) && =delete; - /** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_inline simdjson_result parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = true) & noexcept; - simdjson_inline simdjson_result parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = true) && =delete; - /** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_inline simdjson_result parse_into_document(document& doc, const std::string &s) & noexcept; - simdjson_inline simdjson_result parse_into_document(document& doc, const std::string &s) && =delete; - /** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_inline simdjson_result parse_into_document(document& doc, const padded_string &s) & noexcept; - simdjson_inline simdjson_result parse_into_document(document& doc, const padded_string &s) && =delete; - - /** @private We do not want to allow implicit conversion from C string to std::string. */ - simdjson_inline simdjson_result parse_into_document(document& doc, const char *buf) noexcept = delete; - - /** - * Load a file containing many JSON documents. - * - * dom::parser parser; - * for (const element doc : parser.load_many(path)) { - * cout << std::string(doc["title"]) << endl; - * } - * - * The file is loaded in memory and can be safely deleted after the `parser.load_many(path)` - * function has returned. The memory is held by the `parser` instance. - * - * The function is lazy: it may be that no more than one JSON document at a time is parsed. - * And, possibly, no document many have been parsed when the `parser.load_many(path)` function - * returned. - * - * If there is a UTF-8 BOM, the parser skips it. - * - * ### Format - * - * The file must contain a series of one or more JSON documents, concatenated into a single - * buffer, separated by whitespace. It effectively parses until it has a fully valid document, - * then starts parsing the next document at that point. (It does this with more parallelism and - * lookahead than you might think, though.) - * - * Documents that consist of an object or array may omit the whitespace between them, concatenating - * with no separator. documents that consist of a single primitive (i.e. documents that are not - * arrays or objects) MUST be separated with whitespace. - * - * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse. - * Setting batch_size to excessively large or excessively small values may impact negatively the - * performance. - * - * ### Error Handling - * - * All errors are returned during iteration: if there is a global error such as memory allocation, - * it will be yielded as the first result. Iteration always stops after the first error. - * - * As with all other simdjson methods, non-exception error handling is readily available through - * the same interface, requiring you to check the error before using the document: - * - * dom::parser parser; - * dom::document_stream docs; - * auto error = parser.load_many(path).get(docs); - * if (error) { cerr << error << endl; exit(1); } - * for (auto doc : docs) { - * std::string_view title; - * if ((error = doc["title"].get(title)) { cerr << error << endl; exit(1); } - * cout << title << endl; - * } - * - * ### Threads - * - * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the - * hood to do some lookahead. - * - * ### Parser Capacity - * - * If the parser's current capacity is less than batch_size, it will allocate enough capacity - * to handle it (up to max_capacity). - * - * @param path File name pointing at the concatenated JSON to parse. - * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet - * spot is cache-related: small enough to fit in cache, yet big enough to - * parse as many documents as possible in one tight loop. - * Defaults to 1MB (as simdjson::dom::DEFAULT_BATCH_SIZE), which has been a reasonable sweet - * spot in our tests. - * If you set the batch_size to a value smaller than simdjson::dom::MINIMAL_BATCH_SIZE - * (currently 32B), it will be replaced by simdjson::dom::MINIMAL_BATCH_SIZE. - * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: - * - IO_ERROR if there was an error opening or reading the file. - * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. - * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. - * - other json errors if parsing fails. You should not rely on these errors to always the same for the - * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). - */ - inline simdjson_result load_many(std::string_view path, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; - - /** - * Parse a buffer containing many JSON documents. - * - * dom::parser parser; - * for (element doc : parser.parse_many(buf, len)) { - * cout << std::string(doc["title"]) << endl; - * } - * - * No copy of the input buffer is made. - * - * The function is lazy: it may be that no more than one JSON document at a time is parsed. - * And, possibly, no document many have been parsed when the `parser.load_many(path)` function - * returned. - * - * The caller is responsabile to ensure that the input string data remains unchanged and is - * not deleted during the loop. In particular, the following is unsafe and will not compile: - * - * auto docs = parser.parse_many("[\"temporary data\"]"_padded); - * // here the string "[\"temporary data\"]" may no longer exist in memory - * // the parser instance may not have even accessed the input yet - * for (element doc : docs) { - * cout << std::string(doc["title"]) << endl; - * } - * - * The following is safe: - * - * auto json = "[\"temporary data\"]"_padded; - * auto docs = parser.parse_many(json); - * for (element doc : docs) { - * cout << std::string(doc["title"]) << endl; - * } - * - * If there is a UTF-8 BOM, the parser skips it. - * - * ### Format - * - * The buffer must contain a series of one or more JSON documents, concatenated into a single - * buffer, separated by whitespace. It effectively parses until it has a fully valid document, - * then starts parsing the next document at that point. (It does this with more parallelism and - * lookahead than you might think, though.) - * - * documents that consist of an object or array may omit the whitespace between them, concatenating - * with no separator. documents that consist of a single primitive (i.e. documents that are not - * arrays or objects) MUST be separated with whitespace. - * - * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse. - * Setting batch_size to excessively large or excessively small values may impact negatively the - * performance. - * - * ### Error Handling - * - * All errors are returned during iteration: if there is a global error such as memory allocation, - * it will be yielded as the first result. Iteration always stops after the first error. - * - * As with all other simdjson methods, non-exception error handling is readily available through - * the same interface, requiring you to check the error before using the document: - * - * dom::parser parser; - * dom::document_stream docs; - * auto error = parser.load_many(path).get(docs); - * if (error) { cerr << error << endl; exit(1); } - * for (auto doc : docs) { - * std::string_view title; - * if ((error = doc["title"].get(title)) { cerr << error << endl; exit(1); } - * cout << title << endl; - * } - * - * ### REQUIRED: Buffer Padding - * - * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what - * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you - * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the - * SIMDJSON_PADDING bytes to avoid runtime warnings. - * - * ### Threads - * - * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the - * hood to do some lookahead. - * - * ### Parser Capacity - * - * If the parser's current capacity is less than batch_size, it will allocate enough capacity - * to handle it (up to max_capacity). - * - * @param buf The concatenated JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes. - * @param len The length of the concatenated JSON. - * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet - * spot is cache-related: small enough to fit in cache, yet big enough to - * parse as many documents as possible in one tight loop. - * Defaults to 1MB, which has been a reasonable sweet spot in our tests. - * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: - * - MEMALLOC if the parser does not have enough capacity and memory allocation fails - * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. - * - other json errors if parsing fails. You should not rely on these errors to always the same for the - * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). - */ - inline simdjson_result parse_many(const uint8_t *buf, size_t len, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; - /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline simdjson_result parse_many(const char *buf, size_t len, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; - /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) - * Returns INSUFFICIENT_PADDING if s lacks SIMDJSON_PADDING bytes of spare capacity (or trailing whitespace). */ - inline simdjson_result parse_many(const std::string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; - inline simdjson_result parse_many(const std::string &&s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) = delete;// unsafe - /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ - inline simdjson_result parse_many(const padded_string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; - inline simdjson_result parse_many(const padded_string &&s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) = delete;// unsafe - /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) - * - * Because padded_string_view guarantees SIMDJSON_PADDING trailing bytes, this - * overload is safe to use with buffers that the caller owns elsewhere (for - * example, a padded_memory_map), with no extra copy. Without this overload, - * passing a padded_string_view would silently bind to the padded_string - * overload via an implicit conversion, allocating and copying the input, and - * -- because that temporary is destroyed at the end of the full-expression -- - * leaving the returned document_stream pointing at freed memory. */ - inline simdjson_result parse_many(const padded_string_view &v, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; - - /** @private We do not want to allow implicit conversion from C string to std::string. */ - simdjson_result parse_many(const char *buf, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept = delete; - - /** - * Parse a stream of JSON documents with explicit format specification. - * - * @param buf The concatenated JSON documents. - * @param len The length of the buffer. - * @param batch_size The batch size to use. - * @param format The stream format. - * @return A stream of documents, or an error. - */ - inline simdjson_result parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) noexcept; - /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */ - inline simdjson_result parse_many(const char *buf, size_t len, size_t batch_size, stream_format format) noexcept; - /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */ - inline simdjson_result parse_many(const std::string &s, size_t batch_size, stream_format format) noexcept; - /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */ - inline simdjson_result parse_many(const padded_string &s, size_t batch_size, stream_format format) noexcept; - /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */ - inline simdjson_result parse_many(const padded_string_view &v, size_t batch_size, stream_format format) noexcept; - /** @private An rvalue input is destroyed at the end of the full-expression, while - * the returned document_stream only holds a pointer to it: iterating the stream would - * then read freed memory. These deleted overloads also catch a std::string_view - * argument, which would otherwise convert implicitly to a padded_string temporary. */ - inline simdjson_result parse_many(const std::string &&s, size_t batch_size, stream_format format) = delete;// unsafe - /** @private @overload parse_many(const std::string &&s, size_t batch_size, stream_format format) */ - inline simdjson_result parse_many(const padded_string &&s, size_t batch_size, stream_format format) = delete;// unsafe - - /** - * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length - * and `max_depth` depth. - * - * @param capacity The new capacity. - * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH. - * @return The error, if there is one. - */ - simdjson_warn_unused inline error_code allocate(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept; - -#ifndef SIMDJSON_DISABLE_DEPRECATED_API - /** - * @private deprecated because it returns bool instead of error_code, which is our standard for - * failures. Use allocate() instead. - * - * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length - * and `max_depth` depth. - * - * @param capacity The new capacity. - * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH. - * @return true if successful, false if allocation failed. - */ - [[deprecated("Use allocate() instead.")]] - simdjson_warn_unused inline bool allocate_capacity(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept; -#endif // SIMDJSON_DISABLE_DEPRECATED_API - /** - * The largest document this parser can support without reallocating. - * - * @return Current capacity, in bytes. - */ - simdjson_inline size_t capacity() const noexcept; - - /** - * The largest document this parser can automatically support. - * - * The parser may reallocate internal buffers as needed up to this amount. - * - * @return Maximum capacity, in bytes. - */ - simdjson_inline size_t max_capacity() const noexcept; - - /** - * The maximum level of nested object and arrays supported by this parser. - * - * @return Maximum depth, in bytes. - */ - simdjson_pure simdjson_inline size_t max_depth() const noexcept; - - /** - * Set max_capacity. This is the largest document this parser can automatically support. - * - * The parser may reallocate internal buffers as needed up to this amount as documents are passed - * to it. - * - * Note: To avoid limiting the memory to an absurd value, such as zero or two bytes, - * iff you try to set max_capacity to a value lower than MINIMAL_DOCUMENT_CAPACITY, - * then the maximal capacity is set to MINIMAL_DOCUMENT_CAPACITY. - * - * This call will not allocate or deallocate, even if capacity is currently above max_capacity. - * - * @param max_capacity The new maximum capacity, in bytes. - */ - simdjson_inline void set_max_capacity(size_t max_capacity) noexcept; - -#ifdef SIMDJSON_THREADS_ENABLED - /** - * The parser instance can use threads when they are available to speed up some - * operations. It is enabled by default. Changing this attribute will change the - * behavior of the parser for future operations. Set to true by default. - */ - bool threaded{true}; -#else - /** - * When SIMDJSON_THREADS_ENABLED is not defined, the parser instance cannot use threads. - */ - bool threaded{false}; -#endif - /** @private Use the new DOM API instead */ - class Iterator; - /** @private Use simdjson_error instead */ - using InvalidJSON [[deprecated("Use simdjson_error instead")]] = simdjson_error; - - /** @private [for benchmarking access] The implementation to use */ - std::unique_ptr implementation{}; - - /** @private Use `if (parser.parse(...).error())` instead */ - bool valid{false}; - /** @private Use `parser.parse(...).error()` instead */ - error_code error{UNINITIALIZED}; - - /** @private Use `parser.parse(...).value()` instead */ - document doc{}; - - /** @private returns true if the document parsed was valid */ - [[deprecated("Use the result of parser.parse() instead")]] - inline bool is_valid() const noexcept; - - /** - * @private return an error code corresponding to the last parsing attempt, see - * simdjson.h will return UNINITIALIZED if no parsing was attempted - */ - [[deprecated("Use the result of parser.parse() instead")]] - inline int get_error_code() const noexcept; - - /** @private return the string equivalent of "get_error_code" */ - [[deprecated("Use error_message() on the result of parser.parse() instead, or cout << error")]] - inline std::string get_error_message() const noexcept; - - /** @private */ - [[deprecated("Use cout << on the result of parser.parse() instead")]] - inline bool print_json(std::ostream &os) const noexcept; - - /** @private Private and deprecated: use `parser.parse(...).doc.dump_raw_tape()` instead */ - inline bool dump_raw_tape(std::ostream &os) const noexcept; - - - /** - * When enabled, big integers (exceeding uint64 range) are stored as strings - * in the tape instead of returning BIGINT_ERROR. Default: false. - */ - inline void number_as_string(bool enabled) noexcept { _number_as_string = enabled; } - inline bool number_as_string() const noexcept { return _number_as_string; } - -private: - /** - * The maximum document length this parser will automatically support. - * - * The parser will not be automatically allocated above this amount. - */ - size_t _max_capacity; - - /** Whether to store big integers as strings instead of returning BIGINT_ERROR */ - bool _number_as_string{false}; - - /** - * The loaded buffer (reused each time load() is called) - */ - std::unique_ptr loaded_bytes; - - /** Capacity of loaded_bytes buffer. */ - size_t _loaded_bytes_capacity{0}; - - // all nodes are stored on the doc.tape using a 64-bit word. - // - // strings, double and ints are stored as - // a 64-bit word with a pointer to the actual value - // - // - // - // for objects or arrays, store [ or { at the beginning and } and ] at the - // end. For the openings ([ or {), we annotate them with a reference to the - // location on the doc.tape of the end, and for then closings (} and ]), we - // annotate them with a reference to the location of the opening - // - // - - /** - * Ensure we have enough capacity to handle at least desired_capacity bytes, - * and auto-allocate if not. This also allocates memory if needed in the - * internal document. - */ - inline error_code ensure_capacity(size_t desired_capacity) noexcept; - /** - * Ensure we have enough capacity to handle at least desired_capacity bytes, - * and auto-allocate if not. This also allocates memory if needed in the - * provided document. - */ - inline error_code ensure_capacity(document& doc, size_t desired_capacity) noexcept; - - /** Read the file into loaded_bytes */ - inline simdjson_result read_file(std::string_view path) noexcept; - - friend class parser::Iterator; - friend class document_stream; - - -}; // class parser - -} // namespace dom -} // namespace simdjson - -#endif // SIMDJSON_DOM_PARSER_H -/* end file simdjson/dom/parser.h */ - -#ifdef SIMDJSON_THREADS_ENABLED -#include -#include -#include -#endif - -namespace simdjson { -namespace dom { - -#ifdef SIMDJSON_THREADS_ENABLED -/** @private Custom worker class **/ -struct stage1_worker { - stage1_worker() noexcept = default; - stage1_worker(const stage1_worker&) = delete; - stage1_worker(stage1_worker&&) = delete; - stage1_worker operator=(const stage1_worker&) = delete; - ~stage1_worker(); - /** - * We only start the thread when it is needed, not at object construction, this may throw. - * You should only call this once. - **/ - void start_thread(); - /** - * Start a stage 1 job. You should first call 'run', then 'finish'. - * You must call start_thread once before. - */ - void run(document_stream * ds, dom::parser * stage1, size_t next_batch_start); - /** Wait for the run to finish (blocking). You should first call 'run', then 'finish'. **/ - void finish(); - -private: - - /** - * Normally, we would never stop the thread. But we do in the destructor. - * This function is only safe assuming that you are not waiting for results. You - * should have called run, then finish, and be done. - **/ - void stop_thread(); - - std::thread thread{}; - /** These three variables define the work done by the thread. **/ - dom::parser * stage1_thread_parser{}; - size_t _next_batch_start{}; - document_stream * owner{}; - /** - * We have two state variables. This could be streamlined to one variable in the future but - * we use two for clarity. - */ - bool has_work{false}; - bool can_work{true}; - - /** - * We lock using a mutex. - */ - std::mutex locking_mutex{}; - std::condition_variable cond_var{}; -}; -#endif - -/** - * A forward-only stream of documents. - * - * Produced by parser::parse_many. - * - */ -class document_stream { -public: - /** - * Construct an uninitialized document_stream. - * - * ```cpp - * document_stream docs; - * error = parser.parse_many(json).get(docs); - * ``` - */ - simdjson_inline document_stream() noexcept; - /** Move one document_stream to another. */ - simdjson_inline document_stream(document_stream &&other) noexcept = default; - /** Move one document_stream to another. */ - simdjson_inline document_stream &operator=(document_stream &&other) noexcept = default; - - simdjson_inline ~document_stream() noexcept; - /** - * Returns the input size in bytes. - */ - inline size_t size_in_bytes() const noexcept; - /** - * After iterating through the stream, this method - * returns the number of bytes that were not parsed at the end - * of the stream. If truncated_bytes() differs from zero, - * then the input was truncated maybe because incomplete JSON - * documents were found at the end of the stream. You - * may need to process the bytes in the interval [size_in_bytes()-truncated_bytes(), size_in_bytes()). - * - * You should only call truncated_bytes() after streaming through all - * documents, like so: - * - * document_stream stream = parser.parse_many(json,window); - * for(auto doc : stream) { - * // do something with doc - * } - * size_t truncated = stream.truncated_bytes(); - * - * IMPORTANT: this value is only meaningful under the conditions below. It is - * computed from stage-1 bookkeeping, and outside these conditions it is not - * merely imprecise, it is - * arbitrary -- it can exceed size_in_bytes() or wrap around to a huge value. - * Check it only when all of the following hold: - * - * - you iterated all the way to the end of the stream; - * - no document reported an error. Iteration stops at the first failed - * document, which can leave the bookkeeping from a mid-stream batch. - * - * If you need to know about a truncated tail outside those conditions, track - * it yourself from the last successful document (see iterator::current_index() - * and iterator::source()). - * - * An empty input (zero bytes) or an input made only of white space contains - * no document: truncated_bytes() returns zero. - */ - inline size_t truncated_bytes() const noexcept; - /** - * An iterator through a forward-only stream of documents. - */ - class iterator { - public: - using value_type = simdjson_result; - using reference = value_type; - - using difference_type = std::ptrdiff_t; - - using iterator_category = std::input_iterator_tag; - - /** - * Default constructor. - */ - simdjson_inline iterator() noexcept; - /** - * Get the current document (or error). - */ - simdjson_inline reference operator*() noexcept; - /** - * Advance to the next document (prefix). - */ - inline iterator& operator++() noexcept; - /** - * Check if we're at the end yet. - * @param other the end iterator to compare to. - */ - simdjson_inline bool operator!=(const iterator &other) const noexcept; - /** - * @private - * - * Gives the current index in the input document in bytes. - * - * document_stream stream = parser.parse_many(json,window); - * for(auto i = stream.begin(); i != stream.end(); ++i) { - * auto doc = *i; - * size_t index = i.current_index(); - * } - * - * This function (current_index()) is experimental and the usage - * may change in future versions of simdjson: we find the API somewhat - * awkward and we would like to offer something friendlier. - */ - simdjson_inline size_t current_index() const noexcept; - /** - * @private - * - * Gives a view of the current document. - * - * document_stream stream = parser.parse_many(json,window); - * for(auto i = stream.begin(); i != stream.end(); ++i) { - * auto doc = *i; - * std::string_view v = i->source(); - * } - * - * The returned string_view instance is simply a map to the (unparsed) - * source string: it may thus include white-space characters and all manner - * of padding. - * - * This function (source()) is experimental and the usage - * may change in future versions of simdjson: we find the API somewhat - * awkward and we would like to offer something friendlier. - */ - simdjson_inline std::string_view source() const noexcept; - - private: - simdjson_inline iterator(document_stream *s, bool finished) noexcept; - /** The document_stream we're iterating through. */ - document_stream* stream; - /** Whether we're finished or not. */ - bool finished; - friend class document_stream; - }; - - /** - * Start iterating the documents in the stream. - */ - simdjson_inline iterator begin() noexcept; - /** - * The end of the stream, for iterator comparison purposes. - */ - simdjson_inline iterator end() noexcept; - -private: - - document_stream &operator=(const document_stream &) = delete; // Disallow copying - document_stream(const document_stream &other) = delete; // Disallow copying - - /** - * Construct a document_stream. Does not allocate or parse anything until the iterator is - * used. - * - * @param parser is a reference to the parser instance used to generate this document_stream - * @param buf is the raw byte buffer we need to process - * @param len is the length of the raw byte buffer in bytes - * @param batch_size is the size of the windows (must be strictly greater or equal to the largest JSON document) - * @param format is the stream format - */ - simdjson_inline document_stream( - dom::parser &parser, - const uint8_t *buf, - size_t len, - size_t batch_size, - stream_format format = stream_format::whitespace_delimited - ) noexcept; - - /** - * Parse the first document in the buffer. Used by begin(), to handle allocation and - * initialization. - */ - inline void start() noexcept; - - /** - * Parse the next document found in the buffer previously given to document_stream. - * - * The content should be a valid JSON document encoded as UTF-8. If there is a - * UTF-8 BOM, the parser skips it. - * - * You do NOT need to pre-allocate a parser. This function takes care of - * pre-allocating a capacity defined by the batch_size defined when creating the - * document_stream object. - * - * The function returns simdjson::EMPTY if there is no more data to be parsed. - * - * The function returns simdjson::SUCCESS (as integer = 0) in case of success - * and indicates that the buffer has successfully been parsed to the end. - * Every document it contained has been parsed without error. - * - * The function returns an error code from simdjson/simdjson.h in case of failure - * such as simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth; - * the simdjson::error_message function converts these error codes into a string). - * - * You can also check validity by calling parser.is_valid(). The same parser can - * and should be reused for the other documents in the buffer. - */ - inline void next() noexcept; - - /** - * Pass the next batch through stage 1 and return when finished. - * When threads are enabled, this may wait for the stage 1 thread to finish. - */ - inline void load_batch() noexcept; - - /** Get the next document index. */ - inline size_t next_batch_start() const noexcept; - - /** Pass the next batch through stage 1 with the given parser. */ - inline error_code run_stage1(dom::parser &p, size_t batch_start) noexcept; - - /** Pick the stage 2 entry point for the batch that stage 1 just indexed. */ - inline void load_stage2() noexcept; - - dom::parser *parser; - const uint8_t *buf; - size_t len; - size_t batch_size; - /** The stream format. */ - stream_format format; - /** The error (or lack thereof) from the current document. */ - error_code error; - size_t batch_start{0}; - size_t doc_index{}; - /** The stage 2 entry point for the documents of the current batch (see load_stage2). */ - internal::dom_parser_implementation::stage2_next_function_t stage2_next_fn{nullptr}; -#ifdef SIMDJSON_THREADS_ENABLED - /** Indicates whether we use threads. Note that this needs to be a constant during the execution of the parsing. */ - bool use_thread; - - inline void load_from_stage1_thread() noexcept; - - /** Start a thread to run stage 1 on the next batch. */ - inline void start_stage1_thread() noexcept; - - /** Wait for the stage 1 thread to finish and capture the results. */ - inline void finish_stage1_thread() noexcept; - - /** The error returned from the stage 1 thread. */ - error_code stage1_thread_error{UNINITIALIZED}; - /** The thread used to run stage 1 against the next batch in the background. */ - friend struct stage1_worker; - std::unique_ptr worker{new(std::nothrow) stage1_worker()}; - /** - * The parser used to run stage 1 in the background. Will be swapped - * with the regular parser when finished. - */ - dom::parser stage1_thread_parser{}; -#endif // SIMDJSON_THREADS_ENABLED - - friend class dom::parser; - friend struct simdjson_result; - friend struct internal::simdjson_result_base; - -}; // class document_stream - -} // namespace dom - -template<> -struct simdjson_result : public internal::simdjson_result_base { -public: - simdjson_inline simdjson_result() noexcept; ///< @private - simdjson_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_inline simdjson_result(dom::document_stream &&value) noexcept; ///< @private - -#if SIMDJSON_EXCEPTIONS - simdjson_inline dom::document_stream::iterator begin() noexcept(false); - simdjson_inline dom::document_stream::iterator end() noexcept(false); -#else // SIMDJSON_EXCEPTIONS -#ifndef SIMDJSON_DISABLE_DEPRECATED_API - [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] - simdjson_inline dom::document_stream::iterator begin() noexcept; - [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] - simdjson_inline dom::document_stream::iterator end() noexcept; -#endif // SIMDJSON_DISABLE_DEPRECATED_API -#endif // SIMDJSON_EXCEPTIONS -}; // struct simdjson_result - -} // namespace simdjson - -#endif // SIMDJSON_DOCUMENT_STREAM_H -/* end file simdjson/dom/document_stream.h */ -/* skipped duplicate #include "simdjson/dom/document.h" */ -/* including simdjson/dom/element.h: #include "simdjson/dom/element.h" */ -/* begin file simdjson/dom/element.h */ -#ifndef SIMDJSON_DOM_ELEMENT_H -#define SIMDJSON_DOM_ELEMENT_H - -#include - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/array.h" */ - -namespace simdjson { -namespace dom { - -/** - * The actual concrete type of a JSON element - * This is the type it is most easily cast to with get<>. - */ -enum class element_type { - ARRAY = '[', ///< dom::array - OBJECT = '{', ///< dom::object - INT64 = 'l', ///< int64_t - UINT64 = 'u', ///< uint64_t: any integer that fits in uint64_t but *not* int64_t - DOUBLE = 'd', ///< double: Any number with a "." or "e" that fits in double. - STRING = '"', ///< std::string_view - BOOL = 't', ///< bool - NULL_VALUE = 'n', ///< null - /// The BIGINT type is for integers that do not fit in 64 bits. It is only present - // if you set parser.number_as_string(true). - BIGINT = 'Z' ///< std::string_view: big integer stored as raw digit string -}; - -/** - * A JSON element. - * - * References an element in a JSON document, representing a JSON null, boolean, string, number, - * array or object. - */ -class element { -public: - /** Create a new, invalid element. */ - simdjson_inline element() noexcept; - - /** The type of this element. */ - simdjson_inline element_type type() const noexcept; - - /** - * Cast this element to an array. - * - * @returns An object that can be used to iterate the array, or: - * INCORRECT_TYPE if the JSON element is not an array. - */ - inline simdjson_result get_array() const noexcept; - /** - * Cast this element to an object. - * - * @returns An object that can be used to look up or iterate the object's fields, or: - * INCORRECT_TYPE if the JSON element is not an object. - */ - inline simdjson_result get_object() const noexcept; - /** - * Cast this element to a null-terminated C string. - * - * The string is guaranteed to be valid UTF-8. - * - * The length of the string is given by get_string_length(). Because JSON strings - * may contain null characters, it may be incorrect to use strlen to determine the - * string length. - * - * It is possible to get a single string_view instance which represents both the string - * content and its length: see get_string(). - * - * @returns A pointer to a null-terminated UTF-8 string. This string is stored in the parser and will - * be invalidated the next time it parses a document or when it is destroyed. - * Returns INCORRECT_TYPE if the JSON element is not a string. - */ - inline simdjson_result get_c_str() const noexcept; - /** - * Gives the length in bytes of the string. - * - * It is possible to get a single string_view instance which represents both the string - * content and its length: see get_string(). - * - * @returns A string length in bytes. - * Returns INCORRECT_TYPE if the JSON element is not a string. - */ - inline simdjson_result get_string_length() const noexcept; - /** - * Cast this element to a string. - * - * The string is guaranteed to be valid UTF-8. - * - * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next time it - * parses a document or when it is destroyed. - * Returns INCORRECT_TYPE if the JSON element is not a string. - */ - inline simdjson_result get_string() const noexcept; - - #if SIMDJSON_SUPPORTS_CHAR8_T - /** - * Cast this element to a C++20 UTF-8 string. - * - * The string is guaranteed to be valid UTF-8. - * - * @returns A std::u8string_view. The string is stored in the parser and will be invalidated the next time it - * parses a document or when it is destroyed. - * Returns INCORRECT_TYPE if the JSON element is not a string. - */ - inline simdjson_result get_u8string() const noexcept; - #endif - - /** - * Cast this element to a signed integer. - * - * @returns A signed 64-bit integer. - * Returns INCORRECT_TYPE if the JSON element is not an integer, or NUMBER_OUT_OF_RANGE - * if it is negative. - */ - inline simdjson_result get_int64() const noexcept; - /** - * Cast this element to an unsigned integer. - * - * @returns An unsigned 64-bit integer. - * Returns INCORRECT_TYPE if the JSON element is not an integer, or NUMBER_OUT_OF_RANGE - * if it is too large. - */ - inline simdjson_result get_uint64() const noexcept; - /** - * Cast this element to a double floating-point. - * - * @returns A double value. - * Returns INCORRECT_TYPE if the JSON element is not a number. - */ - inline simdjson_result get_double() const noexcept; - /** - * Cast this element to a bool. - * - * @returns A bool value. - * Returns INCORRECT_TYPE if the JSON element is not a boolean. - */ - inline simdjson_result get_bool() const noexcept; - - /** - * Read this element as a big integer (raw digit string). - * - * @returns A string_view of the raw digits, or: - * INCORRECT_TYPE if the JSON element is not a big integer. - */ - inline simdjson_result get_bigint() const noexcept; - - /** - * Whether this element is a json array. - * - * Equivalent to is(). - */ - inline bool is_array() const noexcept; - /** - * Whether this element is a json object. - * - * Equivalent to is(). - */ - inline bool is_object() const noexcept; - /** - * Whether this element is a json string. - * - * Equivalent to is() or is(). - */ - inline bool is_string() const noexcept; - /** - * Whether this element is a json number that fits in a signed 64-bit integer. - * - * Equivalent to is(). - */ - inline bool is_int64() const noexcept; - /** - * Whether this element is a json number that fits in an unsigned 64-bit integer. - * - * Equivalent to is(). - */ - inline bool is_uint64() const noexcept; - /** - * Whether this element is a json number that fits in a double. - * - * Equivalent to is(). - */ - inline bool is_double() const noexcept; - - /** - * Whether this element is a json number. - * - * Both integers and floating points will return true. - */ - inline bool is_number() const noexcept; - - /** - * Whether this element is a json `true` or `false`. - * - * Equivalent to is(). - */ - inline bool is_bool() const noexcept; - /** - * Whether this element is a json `null`. - */ - inline bool is_null() const noexcept; - - /** - * Whether this element is a big integer (number exceeding 64-bit range). - */ - inline bool is_bigint() const noexcept; - - /** - * Tell whether the value can be cast to provided type (T). - * - * Supported types: - * - Boolean: bool - * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char *, std::u8string_view (C++20) - * - Array: dom::array - * - Object: dom::object - * - * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object - */ - template - simdjson_inline bool is() const noexcept; - - /** - * Get the value as the provided type (T). - * - * Supported types: - * - Boolean: bool - * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char *, std::u8string_view (C++20) - * - Array: dom::array - * - Object: dom::object - * - * You may use get_double(), get_bool(), get_uint64(), get_int64(), - * get_object(), get_array() or get_string() instead. - * - * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object - * - * @returns The value cast to the given type, or: - * INCORRECT_TYPE if the value cannot be cast to the given type. - */ - - template - inline simdjson_result get() const noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should - // immediately fail. - static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " - "The supported types are Boolean (bool), numbers (double, uint64_t, int64_t), " - "strings (std::string_view, const char *), arrays (dom::array) and objects (dom::object). " - "We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - "get_object(), get_array() or get_string() instead of the get template."); - } - - /** - * Get the value as the provided type (T). - * - * Supported types: - * - Boolean: bool - * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char *, std::u8string_view (C++20) - * - Array: dom::array - * - Object: dom::object - * - * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object - * - * @param value The variable to set to the value. May not be set if there is an error. - * - * @returns The error that occurred, or SUCCESS if there was no error. - */ - template - simdjson_warn_unused simdjson_inline error_code get(T &value) const noexcept; - - /** - * Get the value as the provided type (T), setting error if it's not the given type. - * - * Supported types: - * - Boolean: bool - * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char *, std::u8string_view (C++20) - * - Array: dom::array - * - Object: dom::object - * - * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object - * - * @param value The variable to set to the given type. value is undefined if there is an error. - * @param error The variable to store the error. error is set to error_code::SUCCEED if there is an error. - */ - template - inline void tie(T &value, error_code &error) && noexcept; - -#if SIMDJSON_EXCEPTIONS - /** - * Read this element as a boolean. - * - * @return The boolean value - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not a boolean. - */ - inline operator bool() const noexcept(false); - - /** - * Read this element as a null-terminated UTF-8 string. - * - * Be mindful that JSON allows strings to contain null characters. - * - * Does *not* convert other types to a string; requires that the JSON type of the element was - * an actual string. - * - * @return The string value. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not a string. - */ - inline explicit operator const char*() const noexcept(false); - - /** - * Read this element as a null-terminated UTF-8 string. - * - * Does *not* convert other types to a string; requires that the JSON type of the element was - * an actual string. - * - * @return The string value. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not a string. - */ - inline operator std::string_view() const noexcept(false); - - /** - * Read this element as an unsigned integer. - * - * @return The integer value. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an integer - * @exception simdjson_error(NUMBER_OUT_OF_RANGE) if the integer does not fit in 64 bits or is negative - */ - inline operator uint64_t() const noexcept(false); - /** - * Read this element as an signed integer. - * - * @return The integer value. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an integer - * @exception simdjson_error(NUMBER_OUT_OF_RANGE) if the integer does not fit in 64 bits - */ - inline operator int64_t() const noexcept(false); - /** - * Read this element as an double. - * - * @return The double value. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not a number - */ - inline operator double() const noexcept(false); - /** - * Read this element as a JSON array. - * - * @return The JSON array. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an array - */ - inline operator array() const noexcept(false); - /** - * Read this element as a JSON object (key/value pairs). - * - * @return The JSON object. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an object - */ - inline operator object() const noexcept(false); - - /** - * Iterate over each element in this array. - * - * @return The beginning of the iteration. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an array - */ - inline dom::array::iterator begin() const noexcept(false); - - /** - * Iterate over each element in this array. - * - * @return The end of the iteration. - * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an array - */ - inline dom::array::iterator end() const noexcept(false); -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the value associated with the given key. - * - * The key will be matched against **unescaped** JSON: - * - * dom::parser parser; - * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 - * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - * - INCORRECT_TYPE if this is not an object - */ - inline simdjson_result operator[](std::string_view key) const noexcept; - - /** - * Get the value associated with the given key. - * - * The key will be matched against **unescaped** JSON: - * - * dom::parser parser; - * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 - * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - * - INCORRECT_TYPE if this is not an object - */ - inline simdjson_result operator[](const char *key) const noexcept; - simdjson_result operator[](int) const noexcept = delete; - - - /** - * Get the value associated with the given JSON pointer. We use the RFC 6901 - * https://tools.ietf.org/html/rfc6901 standard. - * - * dom::parser parser; - * element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded); - * doc.at_pointer("/foo/a/1") == 20 - * doc.at_pointer("/foo")["a"].at(1) == 20 - * doc.at_pointer("")["foo"]["a"].at(1) == 20 - * - * It is allowed for a key to be the empty string: - * - * dom::parser parser; - * object obj = parser.parse(R"({ "": { "a": [ 10, 20, 30 ] }})"_padded); - * obj.at_pointer("//a/1") == 20 - * - * @return The value associated with the given JSON pointer, or: - * - NO_SUCH_FIELD if a field does not exist in an object - * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length - * - INCORRECT_TYPE if a non-integer is used to access an array - * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed - */ - inline simdjson_result at_pointer(const std::string_view json_pointer) const noexcept; - - inline simdjson_result> at_path_with_wildcard(const std::string_view json_path) const noexcept; - - /** - * Get the value associated with the given JSONPath expression. We only support - * JSONPath queries that trivially convertible to JSON Pointer queries: key - * names and array indices. - * - * https://www.rfc-editor.org/rfc/rfc9535 (RFC 9535) - * - * @return The value associated with the given JSONPath expression, or: - * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails - * - NO_SUCH_FIELD if a field does not exist in an object - * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length - * - INCORRECT_TYPE if a non-integer is used to access an array - */ - inline simdjson_result at_path(std::string_view json_path) const noexcept; - -#ifndef SIMDJSON_DISABLE_DEPRECATED_API - /** - * - * Version 0.4 of simdjson used an incorrect interpretation of the JSON Pointer standard - * and allowed the following : - * - * dom::parser parser; - * element doc = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded); - * doc.at("foo/a/1") == 20 - * - * Though it is intuitive, it is not compliant with RFC 6901 - * https://tools.ietf.org/html/rfc6901 - * - * For standard compliance, use the at_pointer function instead. - * - * @return The value associated with the given JSON pointer, or: - * - NO_SUCH_FIELD if a field does not exist in an object - * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length - * - INCORRECT_TYPE if a non-integer is used to access an array - * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed - */ - [[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] - inline simdjson_result at(const std::string_view json_pointer) const noexcept; -#endif // SIMDJSON_DISABLE_DEPRECATED_API - - /** - * Get the value at the given index. - * - * @return The value at the given index, or: - * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length - */ - inline simdjson_result at(size_t index) const noexcept; - - /** - * Get the value associated with the given key. - * - * The key will be matched against **unescaped** JSON: - * - * dom::parser parser; - * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 - * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - */ - inline simdjson_result at_key(std::string_view key) const noexcept; - - /** - * Get the value associated with the given key in a case-insensitive manner. - * - * Note: The key will be matched against **unescaped** JSON. - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - */ - inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; - - /** - * operator< defines a total order for element allowing to use them in - * ordered C++ STL containers - * - * @return TRUE if the key appears before the other one in the tape - */ - inline bool operator<(const element &other) const noexcept; - - /** - * operator== allows to verify if two element values reference the - * same JSON item - * - * @return TRUE if the two values references the same JSON element - */ - inline bool operator==(const element &other) const noexcept; - - /** @private for debugging. Prints out the root element. */ - inline bool dump_raw_tape(std::ostream &out) const noexcept; - -private: - simdjson_inline element(const internal::tape_ref &tape) noexcept; - internal::tape_ref tape{}; - friend class document; - friend class object; - friend class array; - friend struct simdjson_result; - template - friend class simdjson::internal::string_builder; - -}; - -} // namespace dom - -/** The result of a JSON navigation that may fail. */ -template<> -struct simdjson_result : public internal::simdjson_result_base { -public: - simdjson_inline simdjson_result() noexcept; ///< @private - simdjson_inline simdjson_result(dom::element &&value) noexcept; ///< @private - simdjson_inline simdjson_result(error_code error) noexcept; ///< @private - - simdjson_inline simdjson_result type() const noexcept; - template - simdjson_inline bool is() const noexcept; - template - simdjson_inline simdjson_result get() const noexcept; - template - simdjson_warn_unused simdjson_inline error_code get(T &value) const noexcept; - - simdjson_inline simdjson_result get_array() const noexcept; - simdjson_inline simdjson_result get_object() const noexcept; - simdjson_inline simdjson_result get_c_str() const noexcept; - simdjson_inline simdjson_result get_string_length() const noexcept; - simdjson_inline simdjson_result get_string() const noexcept; - #if SIMDJSON_SUPPORTS_CHAR8_T - simdjson_inline simdjson_result get_u8string() const noexcept; - #endif - simdjson_inline simdjson_result get_int64() const noexcept; - simdjson_inline simdjson_result get_uint64() const noexcept; - simdjson_inline simdjson_result get_double() const noexcept; - simdjson_inline simdjson_result get_bool() const noexcept; - simdjson_inline simdjson_result get_bigint() const noexcept; - - simdjson_inline bool is_array() const noexcept; - simdjson_inline bool is_object() const noexcept; - simdjson_inline bool is_string() const noexcept; - simdjson_inline bool is_int64() const noexcept; - simdjson_inline bool is_uint64() const noexcept; - simdjson_inline bool is_double() const noexcept; - simdjson_inline bool is_number() const noexcept; - simdjson_inline bool is_bool() const noexcept; - simdjson_inline bool is_null() const noexcept; - simdjson_inline bool is_bigint() const noexcept; - - simdjson_inline simdjson_result operator[](std::string_view key) const noexcept; - simdjson_inline simdjson_result operator[](const char *key) const noexcept; - simdjson_result operator[](int) const noexcept = delete; - simdjson_inline simdjson_result at_pointer(const std::string_view json_pointer) const noexcept; - simdjson_inline simdjson_result> at_path_with_wildcard(const std::string_view json_path) const noexcept; - simdjson_inline simdjson_result at_path(const std::string_view json_path) const noexcept; - [[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] - simdjson_inline simdjson_result at(const std::string_view json_pointer) const noexcept; - simdjson_inline simdjson_result at(size_t index) const noexcept; - simdjson_inline simdjson_result at_key(std::string_view key) const noexcept; - simdjson_inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; - -#if SIMDJSON_EXCEPTIONS - simdjson_inline operator bool() const noexcept(false); - simdjson_inline explicit operator const char*() const noexcept(false); - simdjson_inline operator std::string_view() const noexcept(false); - simdjson_inline operator uint64_t() const noexcept(false); - simdjson_inline operator int64_t() const noexcept(false); - simdjson_inline operator double() const noexcept(false); - simdjson_inline operator dom::array() const noexcept(false); - simdjson_inline operator dom::object() const noexcept(false); - - simdjson_inline dom::array::iterator begin() const noexcept(false); - simdjson_inline dom::array::iterator end() const noexcept(false); -#endif // SIMDJSON_EXCEPTIONS -}; - -} // namespace simdjson - -#endif // SIMDJSON_DOM_DOCUMENT_H -/* end file simdjson/dom/element.h */ -/* including simdjson/dom/object.h: #include "simdjson/dom/object.h" */ -/* begin file simdjson/dom/object.h */ -#ifndef SIMDJSON_DOM_OBJECT_H -#define SIMDJSON_DOM_OBJECT_H - -#include - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/element.h" */ -/* skipped duplicate #include "simdjson/internal/tape_ref.h" */ - -namespace simdjson { -namespace dom { - -/** - * JSON object. - */ -class object { -public: - /** Create a new, invalid object */ - simdjson_inline object() noexcept; - - class iterator { - public: - using value_type = const key_value_pair; - using difference_type = std::ptrdiff_t; - using pointer = void; - using reference = value_type; - using iterator_category = std::forward_iterator_tag; - - /** - * Get the actual key/value pair - */ - inline reference operator*() const noexcept; - /** - * Get the next key/value pair. - * - * Part of the std::iterator interface. - * - */ - inline iterator& operator++() noexcept; - /** - * Get the next key/value pair. - * - * Part of the std::iterator interface. - * - */ - inline iterator operator++(int) noexcept; - /** - * Check if these values come from the same place in the JSON. - * - * Part of the std::iterator interface. - */ - inline bool operator!=(const iterator& other) const noexcept; - inline bool operator==(const iterator& other) const noexcept; - - inline bool operator<(const iterator& other) const noexcept; - inline bool operator<=(const iterator& other) const noexcept; - inline bool operator>=(const iterator& other) const noexcept; - inline bool operator>(const iterator& other) const noexcept; - /** - * Get the key of this key/value pair. - */ - inline std::string_view key() const noexcept; - /** - * Get the length (in bytes) of the key in this key/value pair. - * You should expect this function to be faster than key().size(). - */ - inline uint32_t key_length() const noexcept; - /** - * Returns true if the key in this key/value pair is equal - * to the provided string_view. - */ - inline bool key_equals(std::string_view o) const noexcept; - /** - * Returns true if the key in this key/value pair is equal - * to the provided string_view in a case-insensitive manner. - * Case comparisons may only be handled correctly for ASCII strings. - */ - inline bool key_equals_case_insensitive(std::string_view o) const noexcept; - /** - * Get the key of this key/value pair. - */ - inline const char *key_c_str() const noexcept; - /** - * Get the value of this key/value pair. - */ - inline element value() const noexcept; - - iterator() noexcept = default; - iterator(const iterator&) noexcept = default; - iterator& operator=(const iterator&) noexcept = default; - private: - simdjson_inline iterator(const internal::tape_ref &tape) noexcept; - - internal::tape_ref tape{}; - - friend class object; - }; - - /** - * Return the first key/value pair. - * - * Part of the std::iterable interface. - */ - inline iterator begin() const noexcept; - /** - * One past the last key/value pair. - * - * Part of the std::iterable interface. - */ - inline iterator end() const noexcept; - /** - * Get the size of the object (number of keys). - * It is a saturated value with a maximum of 0xFFFFFF: if the value - * is 0xFFFFFF then the size is 0xFFFFFF or greater. - */ - inline size_t size() const noexcept; - /** - * Get the value associated with the given key. - * - * The key will be matched against **unescaped** JSON: - * - * dom::parser parser; - * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 - * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD - * - * This function has linear-time complexity: the keys are checked one by one. - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - * - INCORRECT_TYPE if this is not an object - */ - inline simdjson_result operator[](std::string_view key) const noexcept; - - /** - * Get the value associated with the given key. - * - * The key will be matched against **unescaped** JSON: - * - * dom::parser parser; - * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 - * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD - * - * This function has linear-time complexity: the keys are checked one by one. - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - * - INCORRECT_TYPE if this is not an object - */ - inline simdjson_result operator[](const char *key) const noexcept; - simdjson_result operator[](int) const noexcept = delete; - - /** - * Get the value associated with the given JSON pointer. We use the RFC 6901 - * https://tools.ietf.org/html/rfc6901 standard, interpreting the current node - * as the root of its own JSON document. - * - * dom::parser parser; - * object obj = parser.parse(R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded); - * obj.at_pointer("/foo/a/1") == 20 - * obj.at_pointer("/foo")["a"].at(1) == 20 - * - * It is allowed for a key to be the empty string: - * - * dom::parser parser; - * object obj = parser.parse(R"({ "": { "a": [ 10, 20, 30 ] }})"_padded); - * obj.at_pointer("//a/1") == 20 - * obj.at_pointer("/")["a"].at(1) == 20 - * - * @return The value associated with the given JSON pointer, or: - * - NO_SUCH_FIELD if a field does not exist in an object - * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length - * - INCORRECT_TYPE if a non-integer is used to access an array - * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed - */ - inline simdjson_result at_pointer(std::string_view json_pointer) const noexcept; - - /** - * Recursive function which processes the JSON path of each child element - */ - inline void process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept; - - /** - * Adds support for JSONPath expression with wildcards '*' - */ - inline simdjson_result> at_path_with_wildcard(std::string_view json_path) const noexcept; - - /** - * Get the value associated with the given JSONPath expression. We only support - * JSONPath queries that trivially convertible to JSON Pointer queries: key - * names and array indices. - * - * https://www.rfc-editor.org/rfc/rfc9535 (RFC 9535) - * - * @return The value associated with the given JSONPath expression, or: - * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails - * - NO_SUCH_FIELD if a field does not exist in an object - * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length - * - INCORRECT_TYPE if a non-integer is used to access an array - */ - inline simdjson_result at_path(std::string_view json_path) const noexcept; - - /** - * Get the value associated with the given key. - * - * The key will be matched against **unescaped** JSON: - * - * dom::parser parser; - * int64_t(parser.parse(R"({ "a\n": 1 })"_padded)["a\n"]) == 1 - * parser.parse(R"({ "a\n": 1 })"_padded)["a\\n"].get_uint64().error() == NO_SUCH_FIELD - * - * This function has linear-time complexity: the keys are checked one by one. - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - */ - inline simdjson_result at_key(std::string_view key) const noexcept; - - /** - * Gets the values associated with keys of an object - * This function has linear-time complexity: the keys are checked one by one. - * - * @return the values associated with each key of an object - */ - inline std::vector& get_values(std::vector& out) const noexcept; - - /** - * Get the value associated with the given key in a case-insensitive manner. - * It is only guaranteed to work over ASCII inputs. - * - * Note: The key will be matched against **unescaped** JSON. - * - * This function has linear-time complexity: the keys are checked one by one. - * - * @return The value associated with this field, or: - * - NO_SUCH_FIELD if the field does not exist in the object - */ - inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; - - /** - * Implicitly convert object to element - */ - inline operator element() const noexcept; - -private: - simdjson_inline object(const internal::tape_ref &tape) noexcept; - - internal::tape_ref tape{}; - - friend class element; - friend struct simdjson_result; - template - friend class simdjson::internal::string_builder; -}; - -/** - * Key/value pair in an object. - */ -class key_value_pair { -public: - /** key in the key-value pair **/ - std::string_view key; - /** value in the key-value pair **/ - element value; - -private: - simdjson_inline key_value_pair(std::string_view _key, element _value) noexcept; - friend class object; -}; - -} // namespace dom - -/** The result of a JSON conversion that may fail. */ -template<> -struct simdjson_result : public internal::simdjson_result_base { -public: - simdjson_inline simdjson_result() noexcept; ///< @private - simdjson_inline simdjson_result(dom::object value) noexcept; ///< @private - simdjson_inline simdjson_result(error_code error) noexcept; ///< @private - - inline simdjson_result operator[](std::string_view key) const noexcept; - inline simdjson_result operator[](const char *key) const noexcept; - simdjson_result operator[](int) const noexcept = delete; - inline simdjson_result at_pointer(std::string_view json_pointer) const noexcept; - inline simdjson_result> at_path_with_wildcard(std::string_view json_path_new) const noexcept; - inline simdjson_result at_path(std::string_view json_path) const noexcept; - inline simdjson_result at_key(std::string_view key) const noexcept; - inline std::vector& get_values(std::vector& out) const noexcept; - inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; - -#if SIMDJSON_EXCEPTIONS - inline dom::object::iterator begin() const noexcept(false); - inline dom::object::iterator end() const noexcept(false); - inline size_t size() const noexcept(false); -#endif // SIMDJSON_EXCEPTIONS -}; - -} // namespace simdjson - -#if SIMDJSON_SUPPORTS_RANGES -namespace std { -namespace ranges { -template<> -inline constexpr bool enable_view = true; -#if SIMDJSON_EXCEPTIONS -template<> -inline constexpr bool enable_view> = true; -#endif // SIMDJSON_EXCEPTIONS -} // namespace ranges -} // namespace std -#endif // SIMDJSON_SUPPORTS_RANGES - -#endif // SIMDJSON_DOM_OBJECT_H -/* end file simdjson/dom/object.h */ -/* skipped duplicate #include "simdjson/dom/parser.h" */ -/* including simdjson/dom/serialization.h: #include "simdjson/dom/serialization.h" */ -/* begin file simdjson/dom/serialization.h */ -#ifndef SIMDJSON_SERIALIZATION_H -#define SIMDJSON_SERIALIZATION_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/element.h" */ -/* skipped duplicate #include "simdjson/dom/object.h" */ - -#include - -namespace simdjson { - -/** - * The string_builder template and mini_formatter class - * are not part of our public API and are subject to change - * at any time! - */ -namespace internal { - -template class base_formatter { -public: - /** Add a comma **/ - simdjson_inline void comma(); - /** Start an array, prints [ **/ - simdjson_inline void start_array(); - /** End an array, prints ] **/ - simdjson_inline void end_array(); - /** Start an array, prints { **/ - simdjson_inline void start_object(); - /** Start an array, prints } **/ - simdjson_inline void end_object(); - /** Prints a true **/ - simdjson_inline void true_atom(); - /** Prints a false **/ - simdjson_inline void false_atom(); - /** Prints a null **/ - simdjson_inline void null_atom(); - /** Prints a number **/ - simdjson_inline void number(int64_t x); - /** Prints a number **/ - simdjson_inline void number(uint64_t x); - /** Prints a number **/ - simdjson_inline void number(double x); - /** Prints a key (string + colon) **/ - simdjson_inline void key(std::string_view unescaped); - /** Prints a string. The string is escaped as needed. **/ - simdjson_inline void string(std::string_view unescaped); - /** Clears out the content. **/ - simdjson_inline void clear(); - /** - * Get access to the buffer, it is owned by the instance, but - * the user can make a copy. - **/ - simdjson_inline std::string_view str() const; - - /** Prints one character **/ - simdjson_inline void one_char(char c); - - /** Prints characters in [begin, end) verbatim. **/ - simdjson_inline void chars(const char *begin, const char *end); - - simdjson_inline void call_print_newline() { - static_cast(this)->print_newline(); - } - - simdjson_inline void call_print_indents(size_t depth) { - static_cast(this)->print_indents(depth); - } - - simdjson_inline void call_print_space() { - static_cast(this)->print_space(); - } - -protected: - // implementation details (subject to change) - /** Backing buffer **/ - struct vector_with_small_buffer { - vector_with_small_buffer() = default; - ~vector_with_small_buffer() { free_buffer(); } - - vector_with_small_buffer(const vector_with_small_buffer &) = delete; - vector_with_small_buffer & - operator=(const vector_with_small_buffer &) = delete; - - void clear() { - size = 0; - capacity = StaticCapacity; - free_buffer(); - buffer = array; - } - - simdjson_inline void push_back(char c) { - if (capacity < size + 1) - grow(capacity * 2); - buffer[size++] = c; - } - - /** Drops the space characters at the end of the buffer, if any. **/ - simdjson_inline void trim_trailing_spaces() { - while (size > 0 && buffer[size - 1] == ' ') - size--; - } - - simdjson_inline void append(const char *begin, const char *end) { - const size_t new_size = size + (end - begin); - if (capacity < new_size) - // std::max(new_size, capacity * 2); is broken in tests on Windows - grow(new_size < capacity * 2 ? capacity * 2 : new_size); - std::copy(begin, end, buffer + size); - size = new_size; - } - - simdjson_inline void append_n(char c, size_t count) { - const size_t new_size = size + count; - if (capacity < new_size) { - grow(new_size < capacity * 2 ? capacity * 2 : new_size); - } - std::memset(buffer + size, c, count); - size = new_size; - } - - /** Makes room for n more bytes and returns where they go. **/ - simdjson_inline char *reserve(size_t n) { - if (capacity - size < n) { - grow(size + n < capacity * 2 ? capacity * 2 : size + n); - } - return buffer + size; - } - - /** Ends the content at end, which reserve() returned plus at most n. **/ - simdjson_inline void set_end(const char *end) { size = size_t(end - buffer); } - - std::string_view str() const { return std::string_view(buffer, size); } - - private: - void free_buffer() { - if (buffer != array) - delete[] buffer; - } - void grow(size_t new_capacity) { - auto new_buffer = new char[new_capacity]; - std::copy(buffer, buffer + size, new_buffer); - free_buffer(); - buffer = new_buffer; - capacity = new_capacity; - } - - static const size_t StaticCapacity = 64; - char array[StaticCapacity]; - char *buffer = array; - size_t size = 0; - size_t capacity = StaticCapacity; - } buffer{}; -}; - -/** - * @private This is the class that we expect to use with the string_builder - * template. It tries to produce a compact version of the JSON element - * as quickly as possible. - */ -class mini_formatter : public base_formatter { -public: - simdjson_inline void print_newline(); - - simdjson_inline void print_indents(size_t depth); - - simdjson_inline void print_space(); -}; - -class pretty_formatter : public base_formatter { -public: - explicit pretty_formatter(int step = 4) : indent_step(step) {} - - simdjson_inline void print_newline(); - - simdjson_inline void print_indents(size_t depth); - - simdjson_inline void print_space(); - -protected: - int indent_step = 4; -}; - -/** - * @private The string_builder template allows us to construct - * a string from a document element. It is parametrized - * by a "formatter" which handles the details. Thus - * the string_builder template could support both minification - * and prettification, and various other tradeoffs. - * - * This is not to be confused with the simdjson::builder::string_builder - * which is a different class. - */ -template class string_builder { -public: - /** Construct an initially empty builder, would print the empty string **/ - string_builder() = default; - /** Construct a builder whose formatter uses the given indentation step **/ - explicit string_builder(int indent_step) : format(indent_step) {} - - /** Append an element to the builder (to be printed) **/ - inline void append(simdjson::dom::element value); - /** Append an array to the builder (to be printed) **/ - inline void append(simdjson::dom::array value); - /** Append an object to the builder (to be printed) **/ - inline void append(simdjson::dom::object value); - /** Reset the builder (so that it would print the empty string) **/ - simdjson_inline void clear(); - /** - * Get access to the string. The string_view is owned by the builder - * and it is invalid to use it after the string_builder has been - * destroyed. - * However you can make a copy of the string_view on memory that you - * own. - */ - simdjson_inline std::string_view str() const; - /** Append a key_value_pair to the builder (to be printed) **/ - simdjson_inline void append(simdjson::dom::key_value_pair value); - -private: - /** - * Append an element nested base_depth levels deep: the caller has already - * written the indentation (and key) before it. - **/ - inline void append(simdjson::dom::element value, size_t base_depth); - formatter format{}; -}; - -} // namespace internal - -namespace dom { - -/** - * Print JSON to an output stream. - * - * @param out The output stream. - * @param value The element. - * @throw if there is an error with the underlying output stream. simdjson - * itself will not throw. - */ -inline std::ostream &operator<<(std::ostream &out, - simdjson::dom::element value); -#if SIMDJSON_EXCEPTIONS -inline std::ostream & -operator<<(std::ostream &out, - simdjson::simdjson_result x); -#endif -/** - * Print JSON to an output stream. - * - * @param out The output stream. - * @param value The array. - * @throw if there is an error with the underlying output stream. simdjson - * itself will not throw. - */ -inline std::ostream &operator<<(std::ostream &out, simdjson::dom::array value); -#if SIMDJSON_EXCEPTIONS -inline std::ostream & -operator<<(std::ostream &out, - simdjson::simdjson_result x); -#endif -/** - * Print JSON to an output stream. - * - * @param out The output stream. - * @param value The object. - * @throw if there is an error with the underlying output stream. simdjson - * itself will not throw. - */ -inline std::ostream &operator<<(std::ostream &out, simdjson::dom::object value); -#if SIMDJSON_EXCEPTIONS -inline std::ostream & -operator<<(std::ostream &out, - simdjson::simdjson_result x); -#endif -} // namespace dom - -/** - * Converts JSON to a string. - * - * dom::parser parser; - * element doc = parser.parse(" [ 1 , 2 , 3 ] "_padded); - * cout << to_string(doc) << endl; // prints [1,2,3] - * - */ -template std::string to_string(T x) { - // in C++, to_string is standard: - // http://www.cplusplus.com/reference/string/to_string/ Currently minify and - // to_string are identical but in the future, they may differ. - simdjson::internal::string_builder<> sb; - sb.append(x); - std::string_view answer = sb.str(); - return std::string(answer.data(), answer.size()); -} -#if SIMDJSON_EXCEPTIONS -template std::string to_string(simdjson_result x) { - if (x.error()) { - throw simdjson_error(x.error()); - } - return to_string(x.value()); -} -#endif - -/** - * Minifies a JSON element or document, printing the smallest possible valid - * JSON. - * - * dom::parser parser; - * element doc = parser.parse(" [ 1 , 2 , 3 ] "_padded); - * cout << minify(doc) << endl; // prints [1,2,3] - * - */ -template std::string minify(T x) { return to_string(x); } - -#if SIMDJSON_EXCEPTIONS -template std::string minify(simdjson_result x) { - if (x.error()) { - throw simdjson_error(x.error()); - } - return to_string(x.value()); -} -#endif - -/** - * Prettifies a JSON element or document, printing the valid JSON with - * indentation. - * - * dom::parser parser; - * element doc = parser.parse(" [ 1 , 2 , 3 ] "_padded); - * - * // Prints: - * // { - * // [ - * // 1, - * // 2, - * // 3 - * // ] - * // } - * cout << prettify(doc) << endl; - * - */ -template std::string prettify(T x, int indent_step = 4) { - simdjson::internal::string_builder sb(indent_step); - sb.append(x); - std::string_view answer = sb.str(); - return std::string(answer.data(), answer.size()); -} - -#if SIMDJSON_EXCEPTIONS -template std::string prettify(simdjson_result x, int indent_step = 4) { - if (x.error()) { - throw simdjson_error(x.error()); - } - return prettify(x.value(), indent_step); -} -#endif - -} // namespace simdjson - -#endif -/* end file simdjson/dom/serialization.h */ -/* including simdjson/dom/fractured_json.h: #include "simdjson/dom/fractured_json.h" */ -/* begin file simdjson/dom/fractured_json.h */ -#ifndef SIMDJSON_DOM_FRACTURED_JSON_H -#define SIMDJSON_DOM_FRACTURED_JSON_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/element.h" */ - -namespace simdjson { - -/** Specifies where commas should be in table-formatted elements. */ -enum class table_comma_placement { - /** Commas come right after the value */ - before_padding, - /** Commas come after the column padding, so they line up in their own column. */ - after_padding, - /** Commas come right after the value, except for columns of numbers */ - before_padding_except_numbers, -}; - -/** Options for how lists or columns of numbers should be aligned */ -enum class number_list_alignment { - /** Left-aligns numbers */ - left, - /** Right-aligns numbers */ - right, -}; - -/** - * Configuration options for FracturedJson formatting. - * - * FracturedJson intelligently chooses between different layout strategies - * (inline, compact multiline, table, expanded) based on content complexity, - * length, and structure similarity. - */ -struct fractured_json_options { - /** - * Maximum total characters per line (default: 120). - * Content exceeding this will be expanded to multiple lines. - */ - size_t max_total_line_length = 120; - - /** - * Maximum nesting depth for inline rendering (default: 2). - * Elements with complexity exceeding this will be expanded. - * Complexity 0 = scalar, 1 = flat array/object, 2 = one level of nesting. - */ - size_t max_inline_complexity = 2; - - /** - * Maximum complexity for compact array formatting (default: 2). - * Arrays with elements of this complexity or less may have multiple - * items per line. - */ - size_t max_compact_array_complexity = 2; - - /** - * Number of spaces per indentation level (default: 4). - */ - size_t indent_spaces = 4; - - /** - * Forces elements close to the root to always fully expand, regardless of other settings. - * (default: -1). -1 = none; 0 = root node only; 1 = root node and its children; etc. - */ - int always_expand_depth = -1; - - /** - * Enable tabular formatting for arrays of similar objects or arrays - * (default: true). When enabled, the rows of such an array are written one - * per line with their columns aligned. Rows need not have identical keys: - * columns are ordered by the first occurrence of each key, and a row - * missing a key gets blank space in that column. - */ - bool enable_table_format = true; - - /** - * Maximum complexity of each row of a table (default: 2). - * 0 = rows may only be scalars (a single column); 1 = rows may be flat - * arrays/objects; higher values allow deeper nesting. - */ - size_t max_table_row_complexity = 2; - - /** - * Enable compact multiline arrays (default: true). - * When enabled, arrays of simple elements may have multiple items - * per line. - */ - bool enable_compact_multiline = true; - - /** - * Minimum number of items per line for an array to be formatted as a - * compact multiline array (default: 3). - */ - size_t min_compact_array_row_items = 3; - - /** - * Add space inside brackets for containers that hold only scalar values - * (default: false). When true: { "key": "value" }. When false: - * {"key": "value"}. - * @see nested_bracket_padding - */ - bool simple_bracket_padding = false; - - /** - * Add space inside brackets for containers that hold at least one - * nested array/object (default: true). When true: { "a": [1, 2] }. - * When false: {"a": [1, 2]}. - * @see simple_bracket_padding - */ - bool nested_bracket_padding = true; - - /** - * Add space after colons (default: true). - * When true: "key": "value" - * When false: "key":"value" - */ - bool colon_padding = true; - - /** - * Add space after commas in inline content (default: true). - * When true: [1, 2, 3] - * When false: [1,2,3] - */ - bool comma_padding = true; - - /** - * Placement of commas relative to column padding in table-formatted rows - * and compact multiline arrays (default: before_padding_except_numbers). - */ - table_comma_placement comma_placement = table_comma_placement::before_padding_except_numbers; - - /** - * Controls alignment of numbers in table columns or compact multiline arrays - * (default: left). Numbers are always written exactly as in the input. - */ - number_list_alignment number_alignment = number_list_alignment::left; -}; - -/** - * Format JSON using FracturedJson formatting with default options. - * - * FracturedJson produces human-readable yet compact output by intelligently - * choosing between inline, compact multiline, table, and expanded layouts. - * - * dom::parser parser; - * element doc = parser.parse(json_string); - * cout << fractured_json(doc) << endl; - */ -template -std::string fractured_json(T x); - -/** - * Format JSON using FracturedJson formatting with custom options. - * - * dom::parser parser; - * element doc = parser.parse(json_string); - * fractured_json_options opts; - * opts.max_total_line_length = 80; - * cout << fractured_json(doc, opts) << endl; - */ -template -std::string fractured_json(T x, const fractured_json_options& options); - -#if SIMDJSON_EXCEPTIONS -template -std::string fractured_json(simdjson_result x); - -template -std::string fractured_json(simdjson_result x, const fractured_json_options& options); -#endif - -/** - * Format a JSON string using FracturedJson formatting. - * - * This is useful for formatting output from the builder/static reflection API - * or any valid JSON string. - * - * // With static reflection - * MyStruct data = {...}; - * auto minified = simdjson::to_json_string(data); - * auto formatted = simdjson::fractured_json_string(minified.value()); - * - * // Or with any JSON string - * std::string json = R"({"key":"value"})"; - * auto formatted = simdjson::fractured_json_string(json); - */ -inline std::string fractured_json_string(std::string_view json_str); - -/** - * Format a JSON string using FracturedJson formatting with custom options. - */ -inline std::string fractured_json_string(std::string_view json_str, - const fractured_json_options& options); - -} // namespace simdjson - -#endif // SIMDJSON_DOM_FRACTURED_JSON_H -/* end file simdjson/dom/fractured_json.h */ - -// Inline functions -/* including simdjson/dom/array-inl.h: #include "simdjson/dom/array-inl.h" */ -/* begin file simdjson/dom/array-inl.h */ -#ifndef SIMDJSON_ARRAY_INL_H -#define SIMDJSON_ARRAY_INL_H - -#include - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/array.h" */ -/* skipped duplicate #include "simdjson/dom/element.h" */ -/* skipped duplicate #include "simdjson/error-inl.h" */ -/* including simdjson/jsonpathutil.h: #include "simdjson/jsonpathutil.h" */ -/* begin file simdjson/jsonpathutil.h */ -#ifndef SIMDJSON_JSONPATHUTIL_H -#define SIMDJSON_JSONPATHUTIL_H - -/* skipped duplicate #include "simdjson/error.h" */ -#include -/* skipped duplicate #include "simdjson/common_defs.h" */ - -#include -#include -#include - -namespace simdjson { -namespace internal { -/** - * Parses the next JSON Pointer array index token. - * - * The caller passes a pointer fragment with no leading '/', such as "123/foo". - * On success, array_index receives the parsed index and token_length receives - * the number of bytes consumed before the next '/' or the end of the fragment. - */ -simdjson_inline error_code parse_json_pointer_array_index(std::string_view json_pointer, - size_t &array_index, - size_t &token_length) noexcept { - array_index = 0; - token_length = 0; - - for (; token_length < json_pointer.length() && json_pointer[token_length] != '/'; - token_length++) { - uint8_t digit = uint8_t(json_pointer[token_length] - '0'); - // Check for non-digit in array index. If it's there, we're trying to get a field in an object. - if (digit > 9) { - return INCORRECT_TYPE; - } - // 0 followed by other digits is invalid. - if (token_length > 0 && json_pointer[0] == '0') { - return INVALID_JSON_POINTER; - } - if (array_index > - (((std::numeric_limits::max)() - digit) / 10)) { - return INDEX_OUT_OF_BOUNDS; - } - array_index = array_index * 10 + digit; - } - - // Empty string is invalid; so is a "/" with no digits before it. - if (token_length == 0) { - return INVALID_JSON_POINTER; - } - - return SUCCESS; -} - -/** - * Returns true if the error means that a JSONPath segment matched nothing - * (missing field, index out of bounds, value of the wrong type, or a scalar - * reached while path segments remain), as opposed to a document error. - * A wildcard walk skips such branches, as RFC 9535 and DOM's - * at_path_with_wildcard do. - */ -simdjson_inline bool is_matching_failure_error(error_code error) noexcept { - return error == NO_SUCH_FIELD || error == INDEX_OUT_OF_BOUNDS || - error == INCORRECT_TYPE || error == INVALID_JSON_POINTER; -} -} // namespace internal - -/** - * Converts JSONPath to JSON Pointer. - * @param json_path The JSONPath string to be converted. - * @return A string containing the equivalent JSON Pointer. - */ -inline std::string json_path_to_pointer_conversion(std::string_view json_path) { - size_t i = 0; - // if JSONPath starts with $, skip it - // json_path.starts_with('$') requires C++20. - if (!json_path.empty() && json_path.front() == '$') { - i = 1; - } - if (i >= json_path.size() || (json_path[i] != '.' && - json_path[i] != '[')) { - return "-1"; // This is just a sentinel value, the caller should check for this and return an error. - } - - std::string result; - // Reserve space to reduce allocations, adjusting for potential increases due - // to escaping. - result.reserve(json_path.size() * 2); - - while (i < json_path.length()) { - if (json_path[i] == '.') { - result += '/'; - } else if (json_path[i] == '[') { - result += '/'; - ++i; // Move past the '[' - while (i < json_path.length() && json_path[i] != ']') { - if (json_path[i] == '~') { - result += "~0"; - } else if (json_path[i] == '/') { - result += "~1"; - } else { - result += json_path[i]; - } - ++i; - } - if (i == json_path.length() || json_path[i] != ']') { - return "-1"; // Using sentinel value that will be handled as an error by the caller. - } - } else { - if (json_path[i] == '~') { - result += "~0"; - } else if (json_path[i] == '/') { - result += "~1"; - } else { - result += json_path[i]; - } - } - ++i; - } - - return result; -} - -inline std::pair get_next_key_and_json_path(std::string_view& json_path) { - std::string_view key; - - if (json_path.empty()) { - return {key, json_path}; - } - size_t i = 0; - - // if JSONPath starts with $, skip it - if (json_path.front() == '$') { - i = 1; - } - - - if (i < json_path.length() && json_path[i] == '.') { - i += 1; - size_t key_start = i; - - while (i < json_path.length() && json_path[i] != '[' && json_path[i] != '.') { - ++i; - } - - key = json_path.substr(key_start, i - key_start); - } else if ((i + 1 < json_path.size()) && json_path[i] == '[' && - (json_path[i + 1] == '\'' || json_path[i + 1] == '"')) { - // Bracket-quoted key: ['key'] or ["key"]. - // Require a matching closing quote and a following ']'. If either is - // missing, return an empty key and the original path so callers can treat - // this as a parse failure (e.g. INVALID_JSON_POINTER) without advancing. - // Without this check, i += 2 can make i > size() and substr throws - // std::out_of_range, which aborts noexcept callers such as - // at_path_with_wildcard / for_each_at_path_with_wildcard. - const char quote = json_path[i + 1]; - i += 2; - const size_t key_start = i; - while (i < json_path.length() && json_path[i] != quote) { - ++i; - } - if (i >= json_path.length() || // missing closing quote - i + 1 >= json_path.length() || // missing ] - json_path[i + 1] != ']') { - return {key, json_path}; - } - key = json_path.substr(key_start, i - key_start); - i += 2; // past quote and ] - } else if ((i+2 < json_path.size()) && json_path[i] == '[' && json_path[i+1] == '*' && json_path[i+2] == ']') { // i.e [*].additional_keys or [*]["additional_keys"] - key = "*"; - i += 3; - } else if (i < json_path.size() && json_path[i] == '[') { // i.e. [0] or [123] - size_t j = i + 1; - while (j < json_path.size() && json_path[j] >= '0' && json_path[j] <= '9') { - ++j; - } - if (j > i + 1 && j < json_path.size() && json_path[j] == ']') { - key = json_path.substr(i + 1, j - i - 1); - i = j + 1; - } - } - - - return std::make_pair(key, json_path.substr(i)); -} - -namespace internal { -/** - * One level of a JSONPath as the On-Demand for_each_at_path_with_wildcard() - * consumes it. - */ -struct json_path_segment { - // The key or array index at this level, "*" for a wildcard. Empty if the - // path could not be parsed at this level. - std::string_view key{}; - // Whether a '*' occurs in the rest of the path, after this segment. A - // segment that matches nothing before a wildcard yields an empty result - // instead of an error, as in DOM's at_path_with_wildcard. Like DOM, this - // treats a '*' inside a quoted key as a wildcard. - bool wildcard_follows{false}; -}; - -/** - * Splits json_path with get_next_key_and_json_path(), one segment per level, - * as the On-Demand for_each_at_path_with_wildcard() historically did at each - * node. The keys point into json_path, which must outlive the result. - * - * Every node visited at a given depth sees the same remaining path, so we - * split it once up front: re-scanning the remaining path at every node costs - * O(nodes * path length), which made a long path after a wildcard take - * minutes on a large document. - * - * The last segment either has an empty key (the path is invalid there) or - * is the end of the path. - */ -inline std::vector split_json_path(std::string_view json_path) { - std::vector segments; - // The remaining path (always a suffix of json_path) contains a '*' if and - // only if it starts at or before the last one. - const size_t last_star = json_path.rfind('*'); - std::string_view path = json_path; - do { - std::pair key_and_json_path = get_next_key_and_json_path(path); - path = key_and_json_path.second; - const size_t offset = json_path.size() - path.size(); - json_path_segment segment; - segment.key = key_and_json_path.first; - segment.wildcard_follows = last_star != std::string_view::npos && last_star >= offset; - segments.push_back(segment); - if (segment.key.empty()) { break; } - } while (!path.empty()); - return segments; -} - -/** - * A JSONPath as the DOM at_path_with_wildcard() consumes it, parsed once. - * - * Every node visited at a given depth sees the same remaining path, so we - * parse it up front rather than at each node: re-scanning (and converting) - * the remaining path at every node costs O(nodes * path length), which - * made a long path after a couple of wildcards take minutes on a large - * document. - */ -struct wildcard_json_path { - enum class tail_type { invalid, all_values, pointer }; - // One JSON pointer ("/key") per leading segment, "/*" being a wildcard. - std::vector steps{}; - // What to do once the steps are exhausted. - tail_type tail{tail_type::invalid}; - // The rest of the path, as a JSON pointer, when tail == pointer. - std::string tail_pointer{}; -}; - -/** - * Splits json_path the same way the recursive DOM at_path_with_wildcard() - * historically did, one level at a time, but in a single pass. - */ -inline wildcard_json_path parse_wildcard_json_path(std::string_view json_path) { - wildcard_json_path result; - // Each key is consumed by at_pointer, which unescapes ~0/~1, so '~' and - // '/' in the key must be escaped as json_path_to_pointer_conversion does: - // a raw '/' splits the step into two pointer tokens and a raw '~' - // invalidates it, either way addressing the wrong element. - const auto append_key = [](std::string &pointer, std::string_view key) { - pointer += '/'; - for (const char c : key) { - if (c == '~') { - pointer += "~0"; - } else if (c == '/') { - pointer += "~1"; - } else { - pointer += c; - } - } - }; - // The remaining path (always a suffix of json_path) contains a '*' if and - // only if it starts at or before the last one. - const size_t last_star = json_path.rfind('*'); - std::string_view path = json_path; - while (true) { - size_t i = 0; - if (!path.empty() && path.front() == '$') { - i = 1; - } - if (i >= path.size() || (path[i] != '.' && path[i] != '[')) { - result.tail = wildcard_json_path::tail_type::invalid; - return result; - } - const size_t offset = json_path.size() - path.size(); - if (last_star == std::string_view::npos || last_star < offset) { - // Split the rest like the steps, so that a bracket-quoted key (['b'], - // ["b"]) is the key b, not 'b' with its quotes. What we do not split - // (e.g. [b], the key "b") is converted as at_path does. - std::string_view rest = path; - while (!rest.empty()) { - std::pair key_and_json_path = get_next_key_and_json_path(rest); - if (key_and_json_path.first.empty()) { - const std::string pointer = json_path_to_pointer_conversion(rest); - if (pointer == "-1") { - result.tail_pointer = pointer; - } else { - result.tail_pointer += pointer; - } - break; - } - append_key(result.tail_pointer, key_and_json_path.first); - rest = key_and_json_path.second; - } - result.tail = (result.tail_pointer == "-1") - ? wildcard_json_path::tail_type::invalid - : wildcard_json_path::tail_type::pointer; - return result; - } - if ((path.compare(i, 3, "[*]") == 0 && path.size() == i + 3) || - (path.compare(i, 2, ".*") == 0 && path.size() == i + 2)) { - result.tail = wildcard_json_path::tail_type::all_values; - return result; - } - std::pair key_and_json_path = get_next_key_and_json_path(path); - if (key_and_json_path.first.empty()) { - result.tail = wildcard_json_path::tail_type::invalid; - return result; - } - std::string step; - step.reserve(key_and_json_path.first.size() + 1); - append_key(step, key_and_json_path.first); - result.steps.push_back(std::move(step)); - path = key_and_json_path.second; - // The segment above consumed the path. An empty remainder is the node - // that segment selected (at_pointer("")), not a parse error. The check - // at the top of the loop is for a non-empty remainder that does not - // continue with '.' or '['. A '*' inside the final segment ($.a.b*, - // $['b*']) belongs to that segment. - if (path.empty()) { - result.tail = wildcard_json_path::tail_type::pointer; - return result; - } - } -} -} // namespace internal - -} // namespace simdjson -#endif // SIMDJSON_JSONPATHUTIL_H -/* end file simdjson/jsonpathutil.h */ -/* including simdjson/internal/tape_ref-inl.h: #include "simdjson/internal/tape_ref-inl.h" */ -/* begin file simdjson/internal/tape_ref-inl.h */ -#ifndef SIMDJSON_TAPE_REF_INL_H -#define SIMDJSON_TAPE_REF_INL_H - -/* skipped duplicate #include "simdjson/dom/document.h" */ -/* skipped duplicate #include "simdjson/internal/tape_ref.h" */ -/* including simdjson/internal/tape_type.h: #include "simdjson/internal/tape_type.h" */ -/* begin file simdjson/internal/tape_type.h */ -#ifndef SIMDJSON_INTERNAL_TAPE_TYPE_H -#define SIMDJSON_INTERNAL_TAPE_TYPE_H - -namespace simdjson { -namespace internal { - -/** - * The possible types in the tape. - */ -enum class tape_type { - ROOT = 'r', - START_ARRAY = '[', - START_OBJECT = '{', - END_ARRAY = ']', - END_OBJECT = '}', - STRING = '"', - INT64 = 'l', - UINT64 = 'u', - DOUBLE = 'd', - TRUE_VALUE = 't', - FALSE_VALUE = 'f', - NULL_VALUE = 'n', - BIGINT = 'Z' // Big integer stored as string in string buffer -}; // enum class tape_type - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_TAPE_TYPE_H -/* end file simdjson/internal/tape_type.h */ - -#include - -namespace simdjson { -namespace internal { - -constexpr const uint64_t JSON_VALUE_MASK = 0x00FFFFFFFFFFFFFF; -constexpr const uint32_t JSON_COUNT_MASK = 0xFFFFFF; - -// -// tape_ref inline implementation -// -simdjson_inline tape_ref::tape_ref() noexcept : doc{nullptr}, json_index{0} {} -simdjson_inline tape_ref::tape_ref(const dom::document *_doc, size_t _json_index) noexcept : doc{_doc}, json_index{_json_index} {} - - -simdjson_inline bool tape_ref::is_document_root() const noexcept { - return json_index == 1; // should we ever change the structure of the tape, this should get updated. -} -simdjson_inline bool tape_ref::usable() const noexcept { - return doc != nullptr; // when the document pointer is null, this tape_ref is uninitialized (should not be accessed). -} -// Some value types have a specific on-tape word value. It can be faster -// to check the type by doing a word-to-word comparison instead of extracting the -// most significant 8 bits. - -simdjson_inline bool tape_ref::is_double() const noexcept { - constexpr uint64_t tape_double = uint64_t(tape_type::DOUBLE)<<56; - return doc->tape[json_index] == tape_double; -} -simdjson_inline bool tape_ref::is_int64() const noexcept { - constexpr uint64_t tape_int64 = uint64_t(tape_type::INT64)<<56; - return doc->tape[json_index] == tape_int64; -} -simdjson_inline bool tape_ref::is_uint64() const noexcept { - constexpr uint64_t tape_uint64 = uint64_t(tape_type::UINT64)<<56; - return doc->tape[json_index] == tape_uint64; -} -simdjson_inline bool tape_ref::is_false() const noexcept { - constexpr uint64_t tape_false = uint64_t(tape_type::FALSE_VALUE)<<56; - return doc->tape[json_index] == tape_false; -} -simdjson_inline bool tape_ref::is_true() const noexcept { - constexpr uint64_t tape_true = uint64_t(tape_type::TRUE_VALUE)<<56; - return doc->tape[json_index] == tape_true; -} -simdjson_inline bool tape_ref::is_null_on_tape() const noexcept { - constexpr uint64_t tape_null = uint64_t(tape_type::NULL_VALUE)<<56; - return doc->tape[json_index] == tape_null; -} - -inline size_t tape_ref::after_element() const noexcept { - switch (tape_ref_type()) { - case tape_type::START_ARRAY: - case tape_type::START_OBJECT: - return matching_brace_index(); - case tape_type::UINT64: - case tape_type::INT64: - case tape_type::DOUBLE: - return json_index + 2; - default: - return json_index + 1; - } -} -simdjson_inline tape_type tape_ref::tape_ref_type() const noexcept { - return static_cast(doc->tape[json_index] >> 56); -} -simdjson_inline size_t tape_ref::before_element(size_t array_start) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(usable()); - SIMDJSON_DEVELOPMENT_ASSERT(json_index > array_start); - tape_ref previous(doc, json_index - 1); - if (previous.json_index == array_start) { return array_start; } - // An exact numeric marker cannot end an element unless it is itself the - // payload of a number. In that case its header is immediately before it. - if (previous.is_int64() || previous.is_uint64() || previous.is_double()) { - return previous.json_index - 1; - } - tape_ref probe(doc, previous.json_index - 1); - - // Validate both container links before examining its contents. A candidate - // opening tag preceded by an even run of numeric markers is a real tag, - // not a numeric payload. Scan both candidate boundaries together so that a - // forged opening tag inside a nested value cannot cause an unbounded detour. - // For a real container, the opening probe visits preceding siblings. For a - // numeric payload, the other probe does. Stopping at the shorter run bounds - // the work by the array's immediate elements, rather than nested contents. - const auto type = previous.tape_ref_type(); - if (type == tape_type::END_ARRAY || type == tape_type::END_OBJECT) { - const size_t start = previous.matching_brace_index(); - if (start > array_start && start < previous.json_index) { - tape_ref opening(doc, start); - const auto expected = type == tape_type::END_ARRAY - ? tape_type::START_ARRAY : tape_type::START_OBJECT; - if (opening.tape_ref_type() == expected && - opening.matching_brace_index() == previous.json_index + 1) { - tape_ref before_opening(doc, start - 1); - while ((before_opening.is_int64() || before_opening.is_uint64() || - before_opening.is_double()) && - (probe.is_int64() || probe.is_uint64() || probe.is_double())) { - --before_opening.json_index; - --probe.json_index; - } - if (!before_opening.is_int64() && !before_opening.is_uint64() && - !before_opening.is_double() && - (start - before_opening.json_index) % 2 == 1) { - return start; - } - } - } - } - - // Numeric payloads can have ANY bit pattern, including another type's tag. - // A run of exact numeric markers starts with a header, then alternates - // between payload and header. An odd run before this word makes it a payload. - // Subsequent reverse increments through exact-marker payloads take the - // constant-time numeric-payload branch above. - while (probe.is_int64() || probe.is_uint64() || probe.is_double()) { - --probe.json_index; - } - if ((previous.json_index - probe.json_index) % 2 == 0) { - return previous.json_index - 1; - } - - // Once distinguished from numeric payloads, closing container tags link - // directly back to their opening tags. - switch (previous.tape_ref_type()) { - case tape_type::END_ARRAY: - case tape_type::END_OBJECT: - return previous.matching_brace_index(); - default: - return previous.json_index; - } -} -simdjson_inline uint64_t internal::tape_ref::tape_value() const noexcept { - return doc->tape[json_index] & internal::JSON_VALUE_MASK; -} -simdjson_inline uint32_t internal::tape_ref::matching_brace_index() const noexcept { - return uint32_t(doc->tape[json_index]); -} -simdjson_inline uint32_t internal::tape_ref::scope_count() const noexcept { - return uint32_t((doc->tape[json_index] >> 32) & internal::JSON_COUNT_MASK); -} - -template -simdjson_inline T tape_ref::next_tape_value() const noexcept { - static_assert(sizeof(T) == sizeof(uint64_t), "next_tape_value() template parameter must be 64-bit"); - // Though the following is tempting... - // return *reinterpret_cast(&doc->tape[json_index + 1]); - // It is not generally safe. It is safer, and often faster to rely - // on memcpy. Yes, it is uglier, but it is also encapsulated. - T x; - std::memcpy(&x,&doc->tape[json_index + 1],sizeof(uint64_t)); - return x; -} - -simdjson_inline uint32_t internal::tape_ref::get_string_length() const noexcept { - size_t string_buf_index = size_t(tape_value()); - uint32_t len; - std::memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len)); - return len; -} - -simdjson_inline const char * internal::tape_ref::get_c_str() const noexcept { - size_t string_buf_index = size_t(tape_value()); - return reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]); -} - -inline std::string_view internal::tape_ref::get_string_view() const noexcept { - return std::string_view( - get_c_str(), - get_string_length() - ); -} - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_TAPE_REF_INL_H -/* end file simdjson/internal/tape_ref-inl.h */ - -#include - -namespace simdjson { - -// -// simdjson_result inline implementation -// -simdjson_inline simdjson_result::simdjson_result() noexcept - : internal::simdjson_result_base() {} -simdjson_inline simdjson_result::simdjson_result(dom::array value) noexcept - : internal::simdjson_result_base(std::forward(value)) {} -simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept - : internal::simdjson_result_base(error) {} - -#if SIMDJSON_EXCEPTIONS - -inline dom::array::iterator simdjson_result::begin() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.begin(); -} -inline dom::array::iterator simdjson_result::end() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.end(); -} -inline size_t simdjson_result::size() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.size(); -} -inline dom::array::reverse_iterator simdjson_result::rbegin() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.rbegin(); -} -inline dom::array::reverse_iterator simdjson_result::rend() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.rend(); -} - -#endif // SIMDJSON_EXCEPTIONS - -inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) const noexcept { - if (error()) { return error(); } - return first.at_pointer(json_pointer); -} - - inline simdjson_result simdjson_result::at_path(std::string_view json_path) const noexcept { - if (error()) { return error(); } - auto json_pointer = json_path_to_pointer_conversion(json_path); - if (json_pointer == "-1") { return INVALID_JSON_POINTER; } - return at_pointer(json_pointer); - } - -inline simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) const noexcept { - if (error()) { - return error(); - } - return first.at_path_with_wildcard(json_path); -} - -inline simdjson_result simdjson_result::at(size_t index) const noexcept { - if (error()) { return error(); } - return first.at(index); -} - -inline std::vector& simdjson_result::get_values(std::vector& out) const noexcept { - if (error()) { return out; } - return first.get_values(out); -} - -namespace dom { - -// -// array inline implementation -// -simdjson_inline array::array() noexcept : tape{} {} -simdjson_inline array::array(const internal::tape_ref &_tape) noexcept : tape{_tape} {} -inline array::iterator array::begin() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return internal::tape_ref(tape.doc, tape.json_index + 1); -} -inline array::iterator array::end() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return internal::tape_ref(tape.doc, tape.after_element() - 1); -} -inline size_t array::size() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return tape.scope_count(); -} -inline array::reverse_iterator array::rbegin() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); - const internal::tape_ref end_tape(tape.doc, tape.matching_brace_index() - 1); - return reverse_iterator(internal::tape_ref(tape.doc, end_tape.before_element(tape.json_index)), tape.json_index); -} -inline array::reverse_iterator array::rend() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); - return reverse_iterator(tape, tape.json_index); -} -inline size_t array::number_of_slots() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return tape.matching_brace_index() - tape.json_index; -} -inline simdjson_result array::at_pointer(std::string_view json_pointer) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - if(json_pointer.empty()) { // an empty string means that we return the current node - return element(this->tape); // copy the current node - } else if(json_pointer[0] != '/') { // otherwise there is an error - return INVALID_JSON_POINTER; - } - json_pointer = json_pointer.substr(1); - // - means "the append position" or "the element after the end of the array" - // We don't support this, because we're returning a real element, not a position. - if (json_pointer == "-") { return INDEX_OUT_OF_BOUNDS; } - - size_t array_index = 0; - size_t i; - SIMDJSON_TRY(internal::parse_json_pointer_array_index(json_pointer, array_index, i)); - - // Get the child - auto child = array(tape).at(array_index); - // If there is an error, it ends here - if(child.error()) { - return child; - } - // If there is a /, we're not done yet, call recursively. - if (i < json_pointer.length()) { - child = child.at_pointer(json_pointer.substr(i)); - } - return child; -} - -inline simdjson_result array::at_path(std::string_view json_path) const noexcept { - auto json_pointer = json_path_to_pointer_conversion(json_path); - if (json_pointer == "-1") { return INVALID_JSON_POINTER; } - return at_pointer(json_pointer); -} - -inline void array::process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept { - if (current == end) { - return; - } - - simdjson_result> result; - - - for (auto it = current; it != end; ++it) { - std::vector child_result; - auto error = it->at_path_with_wildcard(path_suffix).get(child_result); - if(error) { - continue; - } - accumulator.insert(accumulator.end(), - std::make_move_iterator(child_result.begin()), - std::make_move_iterator(child_result.end())); - } -} - -inline simdjson_result> array::at_path_with_wildcard(std::string_view json_path) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return element(*this).at_path_with_wildcard(json_path); -} - -inline simdjson_result array::at(size_t index) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - size_t i=0; - for (auto element : *this) { - if (i == index) { return element; } - i++; - } - return INDEX_OUT_OF_BOUNDS; -} - -inline std::vector& array::get_values(std::vector& out) const noexcept { - out.reserve(this->size()); - for (auto element : *this) { - out.emplace_back(element); - } - - return out; -} - -inline array::operator element() const noexcept { - return element(tape); -} - -// -// array::reverse_iterator inline implementation -// -simdjson_inline array::reverse_iterator::reverse_iterator(const internal::tape_ref &_tape, size_t _array_start) noexcept - : tape{_tape}, array_start{_array_start} { } -inline element array::reverse_iterator::operator*() const noexcept { - return element(tape); -} -inline array::reverse_iterator& array::reverse_iterator::operator++() noexcept { - tape.json_index = tape.before_element(array_start); - return *this; -} -inline array::reverse_iterator array::reverse_iterator::operator++(int) noexcept { - reverse_iterator out = *this; - ++*this; - return out; -} -inline bool array::reverse_iterator::operator==(const reverse_iterator& other) const noexcept { - return tape.doc == other.tape.doc && tape.json_index == other.tape.json_index; -} -inline bool array::reverse_iterator::operator!=(const reverse_iterator& other) const noexcept { - return !(*this == other); -} - -// -// array::iterator inline implementation -// -simdjson_inline array::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } -inline element array::iterator::operator*() const noexcept { - return element(tape); -} -inline array::iterator& array::iterator::operator++() noexcept { - tape.json_index = tape.after_element(); - return *this; -} -inline array::iterator array::iterator::operator++(int) noexcept { - array::iterator out = *this; - ++*this; - return out; -} -inline bool array::iterator::operator!=(const array::iterator& other) const noexcept { - return tape.json_index != other.tape.json_index; -} -inline bool array::iterator::operator==(const array::iterator& other) const noexcept { - return tape.json_index == other.tape.json_index; -} -inline bool array::iterator::operator<(const array::iterator& other) const noexcept { - return tape.json_index < other.tape.json_index; -} -inline bool array::iterator::operator<=(const array::iterator& other) const noexcept { - return tape.json_index <= other.tape.json_index; -} -inline bool array::iterator::operator>=(const array::iterator& other) const noexcept { - return tape.json_index >= other.tape.json_index; -} -inline bool array::iterator::operator>(const array::iterator& other) const noexcept { - return tape.json_index > other.tape.json_index; -} - -} // namespace dom - - -} // namespace simdjson - -/* including simdjson/dom/element-inl.h: #include "simdjson/dom/element-inl.h" */ -/* begin file simdjson/dom/element-inl.h */ -#ifndef SIMDJSON_ELEMENT_INL_H -#define SIMDJSON_ELEMENT_INL_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/element.h" */ -/* skipped duplicate #include "simdjson/dom/document.h" */ -/* skipped duplicate #include "simdjson/dom/object.h" */ -/* skipped duplicate #include "simdjson/internal/tape_type.h" */ - -/* including simdjson/dom/object-inl.h: #include "simdjson/dom/object-inl.h" */ -/* begin file simdjson/dom/object-inl.h */ -#ifndef SIMDJSON_OBJECT_INL_H -#define SIMDJSON_OBJECT_INL_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/object.h" */ -/* skipped duplicate #include "simdjson/dom/document.h" */ - -/* skipped duplicate #include "simdjson/dom/element-inl.h" */ -/* skipped duplicate #include "simdjson/error-inl.h" */ -/* skipped duplicate #include "simdjson/jsonpathutil.h" */ - -#include - -namespace simdjson { - -// -// simdjson_result inline implementation -// -simdjson_inline simdjson_result::simdjson_result() noexcept - : internal::simdjson_result_base() {} -simdjson_inline simdjson_result::simdjson_result(dom::object value) noexcept - : internal::simdjson_result_base(std::forward(value)) {} -simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept - : internal::simdjson_result_base(error) {} - -inline simdjson_result simdjson_result::operator[](std::string_view key) const noexcept { - if (error()) { return error(); } - return first[key]; -} -inline simdjson_result simdjson_result::operator[](const char *key) const noexcept { - if (error()) { return error(); } - return first[key]; -} -inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) const noexcept { - if (error()) { return error(); } - return first.at_pointer(json_pointer); -} -inline simdjson_result simdjson_result::at_path(std::string_view json_path) const noexcept { - if (error()) { return error(); } - auto json_pointer = json_path_to_pointer_conversion(json_path); - if (json_pointer == "-1") { return INVALID_JSON_POINTER; } - return at_pointer(json_pointer); -} -inline simdjson_result> simdjson_result::at_path_with_wildcard(std::string_view json_path) const noexcept { - if (error()) { - return error(); - } - return first.at_path_with_wildcard(json_path); -} -inline simdjson_result simdjson_result::at_key(std::string_view key) const noexcept { - if (error()) { return error(); } - return first.at_key(key); -} -inline std::vector& simdjson_result::get_values(std::vector& out) const noexcept { - if (error()) { return out; } - return first.get_values(out); -} -inline simdjson_result simdjson_result::at_key_case_insensitive(std::string_view key) const noexcept { - if (error()) { return error(); } - return first.at_key_case_insensitive(key); -} - -#if SIMDJSON_EXCEPTIONS - -inline dom::object::iterator simdjson_result::begin() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.begin(); -} -inline dom::object::iterator simdjson_result::end() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.end(); -} -inline size_t simdjson_result::size() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.size(); -} - -#endif // SIMDJSON_EXCEPTIONS - -namespace dom { - -// -// object inline implementation -// -simdjson_inline object::object() noexcept : tape{} {} -simdjson_inline object::object(const internal::tape_ref &_tape) noexcept : tape{_tape} { } -inline object::iterator object::begin() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return internal::tape_ref(tape.doc, tape.json_index + 1); -} -inline object::iterator object::end() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return internal::tape_ref(tape.doc, tape.after_element() - 1); -} -inline size_t object::size() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return tape.scope_count(); -} - -inline simdjson_result object::operator[](std::string_view key) const noexcept { - return at_key(key); -} -inline simdjson_result object::operator[](const char *key) const noexcept { - return at_key(key); -} -inline simdjson_result object::at_pointer(std::string_view json_pointer) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - if(json_pointer.empty()) { // an empty string means that we return the current node - return element(this->tape); // copy the current node - } else if(json_pointer[0] != '/') { // otherwise there is an error - return INVALID_JSON_POINTER; - } - json_pointer = json_pointer.substr(1); - size_t slash = json_pointer.find('/'); - std::string_view key = json_pointer.substr(0, slash); - // Grab the child with the given key - simdjson_result child; - - // If there is an escape character in the key, unescape it and then get the child. - size_t escape = key.find('~'); - if (escape != std::string_view::npos) { - // Unescape the key - std::string unescaped(key); - do { - switch (unescaped[escape+1]) { - case '0': - unescaped.replace(escape, 2, "~"); - break; - case '1': - unescaped.replace(escape, 2, "/"); - break; - default: - return INVALID_JSON_POINTER; // "Unexpected ~ escape character in JSON pointer"); - } - escape = unescaped.find('~', escape+1); - } while (escape != std::string::npos); - child = at_key(unescaped); - } else { - child = at_key(key); - } - if(child.error()) { - return child; // we do not continue if there was an error - } - // If there is a /, we have to recurse and look up more of the path - if (slash != std::string_view::npos) { - child = child.at_pointer(json_pointer.substr(slash)); - } - return child; -} - -inline simdjson_result object::at_path(std::string_view json_path) const noexcept { - auto json_pointer = json_path_to_pointer_conversion(json_path); - if (json_pointer == "-1") { return INVALID_JSON_POINTER; } - return at_pointer(json_pointer); -} - -inline void object::process_json_path_of_child_elements(std::vector::iterator& current, std::vector::iterator& end, const std::string_view& path_suffix, std::vector& accumulator) const noexcept { - if (current == end) { - return; - } - - simdjson_result> result; - - for (auto it = current; it != end; ++it) { - std::vector child_result; - auto error = it->at_path_with_wildcard(path_suffix).get(child_result); - if(error) { - continue; - } - accumulator.insert(accumulator.end(), - std::make_move_iterator(child_result.begin()), - std::make_move_iterator(child_result.end())); - } -} - -inline simdjson_result> object::at_path_with_wildcard(std::string_view json_path) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return element(*this).at_path_with_wildcard(json_path); -} - -inline simdjson_result object::at_key(std::string_view key) const noexcept { - iterator end_field = end(); - for (iterator field = begin(); field != end_field; ++field) { - if (field.key_equals(key)) { - return field.value(); - } - } - return NO_SUCH_FIELD; -} - -inline std::vector& object::get_values(std::vector& out) const noexcept { - iterator end_field = end(); - iterator begin_field = begin(); - - out.reserve(std::distance(begin_field, end_field)); - for (iterator field = begin_field; field != end_field; ++field) { - out.emplace_back(field.value()); - } - - return out; -} -// In case you wonder why we need this, please see -// https://github.com/simdjson/simdjson/issues/323 -// People do seek keys in a case-insensitive manner. -inline simdjson_result object::at_key_case_insensitive(std::string_view key) const noexcept { - iterator end_field = end(); - for (iterator field = begin(); field != end_field; ++field) { - if (field.key_equals_case_insensitive(key)) { - return field.value(); - } - } - return NO_SUCH_FIELD; -} - -inline object::operator element() const noexcept { - return element(tape); -} - -// -// object::iterator inline implementation -// -simdjson_inline object::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } -inline const key_value_pair object::iterator::operator*() const noexcept { - return key_value_pair(key(), value()); -} -inline bool object::iterator::operator!=(const object::iterator& other) const noexcept { - return tape.json_index != other.tape.json_index; -} -inline bool object::iterator::operator==(const object::iterator& other) const noexcept { - return tape.json_index == other.tape.json_index; -} -inline bool object::iterator::operator<(const object::iterator& other) const noexcept { - return tape.json_index < other.tape.json_index; -} -inline bool object::iterator::operator<=(const object::iterator& other) const noexcept { - return tape.json_index <= other.tape.json_index; -} -inline bool object::iterator::operator>=(const object::iterator& other) const noexcept { - return tape.json_index >= other.tape.json_index; -} -inline bool object::iterator::operator>(const object::iterator& other) const noexcept { - return tape.json_index > other.tape.json_index; -} -inline object::iterator& object::iterator::operator++() noexcept { - tape.json_index++; - tape.json_index = tape.after_element(); - return *this; -} -inline object::iterator object::iterator::operator++(int) noexcept { - object::iterator out = *this; - ++*this; - return out; -} -inline std::string_view object::iterator::key() const noexcept { - return tape.get_string_view(); -} -inline uint32_t object::iterator::key_length() const noexcept { - return tape.get_string_length(); -} -inline const char* object::iterator::key_c_str() const noexcept { - return reinterpret_cast(&tape.doc->string_buf[size_t(tape.tape_value()) + sizeof(uint32_t)]); -} -inline element object::iterator::value() const noexcept { - return element(internal::tape_ref(tape.doc, tape.json_index + 1)); -} - -/** - * Design notes: - * Instead of constructing a string_view and then comparing it with a - * user-provided strings, it is probably more performant to have dedicated - * functions taking as a parameter the string we want to compare against - * and return true when they are equal. That avoids the creation of a temporary - * std::string_view. Though it is possible for the compiler to avoid entirely - * any overhead due to string_view, relying too much on compiler magic is - * problematic: compiler magic sometimes fail, and then what do you do? - * Also, enticing users to rely on high-performance function is probably better - * on the long run. - */ - -inline bool object::iterator::key_equals(std::string_view o) const noexcept { - // We use the fact that the key length can be computed quickly - // without access to the string buffer. - const uint32_t len = key_length(); - if(o.size() == len) { - // We avoid construction of a temporary string_view instance. - return (memcmp(o.data(), key_c_str(), len) == 0); - } - return false; -} - -inline bool object::iterator::key_equals_case_insensitive(std::string_view o) const noexcept { - // We use the fact that the key length can be computed quickly - // without access to the string buffer. - const uint32_t len = key_length(); - if(o.size() == len) { - // See For case-insensitive string comparisons, avoid char-by-char functions - // https://lemire.me/blog/2020/04/30/for-case-insensitive-string-comparisons-avoid-char-by-char-functions/ - // Note that it might be worth rolling our own strncasecmp function, with vectorization. - // - // strncasecmp stops at a null character, and a JSON key may contain some: - // when it does, we resume the comparison right after the null character. - const char *key = key_c_str(); - size_t i = 0; - while (simdjson_strncasecmp(o.data() + i, key + i, len - i) == 0) { - const void *null_char = std::memchr(key + i, '\0', len - i); - if (null_char == nullptr) { return true; } - i = size_t(static_cast(null_char) - key) + 1; - } - } - return false; -} -// -// key_value_pair inline implementation -// -inline key_value_pair::key_value_pair(std::string_view _key, element _value) noexcept : - key(_key), value(_value) {} - -} // namespace dom - -} // namespace simdjson - -#if SIMDJSON_SUPPORTS_RANGES -static_assert(std::ranges::view); -static_assert(std::ranges::sized_range); -#if SIMDJSON_EXCEPTIONS -static_assert(std::ranges::view>); -static_assert(std::ranges::sized_range>); -#endif // SIMDJSON_EXCEPTIONS -#endif // SIMDJSON_SUPPORTS_RANGES - -#endif // SIMDJSON_OBJECT_INL_H -/* end file simdjson/dom/object-inl.h */ -/* skipped duplicate #include "simdjson/error-inl.h" */ -/* skipped duplicate #include "simdjson/jsonpathutil.h" */ - -#include -#include - -namespace simdjson { - -// -// simdjson_result inline implementation -// -simdjson_inline simdjson_result::simdjson_result() noexcept - : internal::simdjson_result_base() {} -simdjson_inline simdjson_result::simdjson_result(dom::element &&value) noexcept - : internal::simdjson_result_base(std::forward(value)) {} -simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept - : internal::simdjson_result_base(error) {} -inline simdjson_result simdjson_result::type() const noexcept { - if (error()) { return error(); } - return first.type(); -} - -template -simdjson_inline bool simdjson_result::is() const noexcept { - return !error() && first.is(); -} -template -simdjson_inline simdjson_result simdjson_result::get() const noexcept { - if (error()) { return error(); } - return first.get(); -} -template -simdjson_warn_unused simdjson_inline error_code simdjson_result::get(T &value) const noexcept { - if (error()) { return error(); } - return first.get(value); -} - -simdjson_inline simdjson_result simdjson_result::get_array() const noexcept { - if (error()) { return error(); } - return first.get_array(); -} -simdjson_inline simdjson_result simdjson_result::get_object() const noexcept { - if (error()) { return error(); } - return first.get_object(); -} -simdjson_inline simdjson_result simdjson_result::get_c_str() const noexcept { - if (error()) { return error(); } - return first.get_c_str(); -} -simdjson_inline simdjson_result simdjson_result::get_string_length() const noexcept { - if (error()) { return error(); } - return first.get_string_length(); -} -simdjson_inline simdjson_result simdjson_result::get_string() const noexcept { - if (error()) { return error(); } - return first.get_string(); -} -#if SIMDJSON_SUPPORTS_CHAR8_T -simdjson_inline simdjson_result simdjson_result::get_u8string() const noexcept { - if (error()) { return error(); } - return first.get_u8string(); -} -#endif -simdjson_inline simdjson_result simdjson_result::get_int64() const noexcept { - if (error()) { return error(); } - return first.get_int64(); -} -simdjson_inline simdjson_result simdjson_result::get_uint64() const noexcept { - if (error()) { return error(); } - return first.get_uint64(); -} -simdjson_inline simdjson_result simdjson_result::get_double() const noexcept { - if (error()) { return error(); } - return first.get_double(); -} -simdjson_inline simdjson_result simdjson_result::get_bool() const noexcept { - if (error()) { return error(); } - return first.get_bool(); -} -simdjson_inline simdjson_result simdjson_result::get_bigint() const noexcept { - if (error()) { return error(); } - return first.get_bigint(); -} - -simdjson_inline bool simdjson_result::is_array() const noexcept { - return !error() && first.is_array(); -} -simdjson_inline bool simdjson_result::is_object() const noexcept { - return !error() && first.is_object(); -} -simdjson_inline bool simdjson_result::is_string() const noexcept { - return !error() && first.is_string(); -} -simdjson_inline bool simdjson_result::is_int64() const noexcept { - return !error() && first.is_int64(); -} -simdjson_inline bool simdjson_result::is_uint64() const noexcept { - return !error() && first.is_uint64(); -} -simdjson_inline bool simdjson_result::is_double() const noexcept { - return !error() && first.is_double(); -} -simdjson_inline bool simdjson_result::is_number() const noexcept { - return !error() && first.is_number(); -} -simdjson_inline bool simdjson_result::is_bool() const noexcept { - return !error() && first.is_bool(); -} - -simdjson_inline bool simdjson_result::is_null() const noexcept { - return !error() && first.is_null(); -} -simdjson_inline bool simdjson_result::is_bigint() const noexcept { - return !error() && first.is_bigint(); -} - -simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) const noexcept { - if (error()) { return error(); } - return first[key]; -} -simdjson_inline simdjson_result simdjson_result::operator[](const char *key) const noexcept { - if (error()) { return error(); } - return first[key]; -} -simdjson_inline simdjson_result simdjson_result::at_pointer(const std::string_view json_pointer) const noexcept { - if (error()) { return error(); } - return first.at_pointer(json_pointer); -} -simdjson_inline simdjson_result simdjson_result::at_path(const std::string_view json_path) const noexcept { - if (error()) { return error(); } - auto json_pointer = json_path_to_pointer_conversion(json_path); - if (json_pointer == "-1") { return INVALID_JSON_POINTER; } - return at_pointer(json_pointer); -} - -simdjson_inline simdjson_result> simdjson_result::at_path_with_wildcard(const std::string_view json_path) const noexcept { - if (error()) { return error(); } - return first.at_path_with_wildcard(json_path); -} - -#ifndef SIMDJSON_DISABLE_DEPRECATED_API -[[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] -simdjson_inline simdjson_result simdjson_result::at(const std::string_view json_pointer) const noexcept { -SIMDJSON_PUSH_DISABLE_WARNINGS -SIMDJSON_DISABLE_DEPRECATED_WARNING - if (error()) { return error(); } - return first.at(json_pointer); -SIMDJSON_POP_DISABLE_WARNINGS -} -#endif // SIMDJSON_DISABLE_DEPRECATED_API -simdjson_inline simdjson_result simdjson_result::at(size_t index) const noexcept { - if (error()) { return error(); } - return first.at(index); -} -simdjson_inline simdjson_result simdjson_result::at_key(std::string_view key) const noexcept { - if (error()) { return error(); } - return first.at_key(key); -} -simdjson_inline simdjson_result simdjson_result::at_key_case_insensitive(std::string_view key) const noexcept { - if (error()) { return error(); } - return first.at_key_case_insensitive(key); -} - -#if SIMDJSON_EXCEPTIONS - -simdjson_inline simdjson_result::operator bool() const noexcept(false) { - return get(); -} -simdjson_inline simdjson_result::operator const char *() const noexcept(false) { - return get(); -} -simdjson_inline simdjson_result::operator std::string_view() const noexcept(false) { - return get(); -} -simdjson_inline simdjson_result::operator uint64_t() const noexcept(false) { - return get(); -} -simdjson_inline simdjson_result::operator int64_t() const noexcept(false) { - return get(); -} -simdjson_inline simdjson_result::operator double() const noexcept(false) { - return get(); -} -simdjson_inline simdjson_result::operator dom::array() const noexcept(false) { - return get(); -} -simdjson_inline simdjson_result::operator dom::object() const noexcept(false) { - return get(); -} - -simdjson_inline dom::array::iterator simdjson_result::begin() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.begin(); -} -simdjson_inline dom::array::iterator simdjson_result::end() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.end(); -} - -#endif // SIMDJSON_EXCEPTIONS - -namespace internal { -// Appends the children of e, which must be an array or an object. -inline void get_container_values(const dom::element &e, std::vector &out) noexcept { - dom::array a; - if (!e.get_array().get(a)) { - a.get_values(out); - return; - } - dom::object o; - if (!e.get_object().get(o)) { - o.get_values(out); - } -} - -// Walks a pre-parsed wildcard path from e, which is at depth `step` of the -// path. Errors below the starting node are skipped, not reported. -inline simdjson_result> at_wildcard_json_path(const dom::element &e, const wildcard_json_path &path, size_t step) noexcept { - // A finished path names this node, even when it is a scalar. Checking the - // container type first would drop keys such as "b*" and "foo*". - if (step == path.steps.size()) { - switch (path.tail) { - case wildcard_json_path::tail_type::all_values: { - std::vector children; - get_container_values(e, children); - return children; - } - case wildcard_json_path::tail_type::pointer: { - dom::element result; - auto error = e.at_pointer(path.tail_pointer).get(result); - if (error) { - return error; - } - return std::vector{std::move(result)}; - } - default: - return INVALID_JSON_POINTER; - } - } - const dom::element_type t = e.type(); - if (t != dom::element_type::ARRAY && t != dom::element_type::OBJECT) { - return std::vector{}; - } - std::vector children; - const std::string &key_pointer = path.steps[step]; - if (key_pointer == "/*") { - get_container_values(e, children); - } else { - dom::element child; - if (!e.at_pointer(key_pointer).get(child)) { - children.emplace_back(child); - } - } - std::vector result; - for (const dom::element &child : children) { - std::vector child_result; - if (at_wildcard_json_path(child, path, step + 1).get(child_result)) { - continue; - } - result.insert(result.end(), - std::make_move_iterator(child_result.begin()), - std::make_move_iterator(child_result.end())); - } - return result; -} -} // namespace internal - -namespace dom { - -// -// element inline implementation -// -simdjson_inline element::element() noexcept : tape{} {} -simdjson_inline element::element(const internal::tape_ref &_tape) noexcept : tape{_tape} { } - -inline element_type element::type() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - auto tape_type = tape.tape_ref_type(); - return tape_type == internal::tape_type::FALSE_VALUE ? element_type::BOOL : static_cast(tape_type); -} - -inline simdjson_result element::get_bool() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - if(tape.is_true()) { - return true; - } else if(tape.is_false()) { - return false; - } - return INCORRECT_TYPE; -} -inline simdjson_result element::get_bigint() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); - switch (tape.tape_ref_type()) { - case internal::tape_type::BIGINT: - return tape.get_string_view(); - default: - return INCORRECT_TYPE; - } -} -inline simdjson_result element::get_c_str() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - switch (tape.tape_ref_type()) { - case internal::tape_type::STRING: { - return tape.get_c_str(); - } - default: - return INCORRECT_TYPE; - } -} -inline simdjson_result element::get_string_length() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - switch (tape.tape_ref_type()) { - case internal::tape_type::STRING: { - return tape.get_string_length(); - } - default: - return INCORRECT_TYPE; - } -} -inline simdjson_result element::get_string() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - switch (tape.tape_ref_type()) { - case internal::tape_type::STRING: - return tape.get_string_view(); - default: - return INCORRECT_TYPE; - } -} -#if SIMDJSON_SUPPORTS_CHAR8_T -inline simdjson_result element::get_u8string() const noexcept { - std::string_view v; - SIMDJSON_TRY(get_string().get(v)); - return internal::as_u8string_view(v); -} -#endif -inline simdjson_result element::get_uint64() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - if(simdjson_unlikely(!tape.is_uint64())) { // branch rarely taken - if(tape.is_int64()) { - int64_t result = tape.next_tape_value(); - if (result < 0) { - return NUMBER_OUT_OF_RANGE; - } - return uint64_t(result); - } - return INCORRECT_TYPE; - } - return tape.next_tape_value(); -} -inline simdjson_result element::get_int64() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - if(simdjson_unlikely(!tape.is_int64())) { // branch rarely taken - if(tape.is_uint64()) { - uint64_t result = tape.next_tape_value(); - // Wrapping max in parens to handle Windows issue: https://stackoverflow.com/questions/11544073/how-do-i-deal-with-the-max-macro-in-windows-h-colliding-with-max-in-std - if (result > uint64_t((std::numeric_limits::max)())) { - return NUMBER_OUT_OF_RANGE; - } - return static_cast(result); - } - return INCORRECT_TYPE; - } - return tape.next_tape_value(); -} -inline simdjson_result element::get_double() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - // Performance considerations: - // 1. Querying tape_ref_type() implies doing a shift, it is fast to just do a straight - // comparison. - // 2. Using a switch-case relies on the compiler guessing what kind of code generation - // we want... But the compiler cannot know that we expect the type to be "double" - // most of the time. - // We can expect get to refer to a double type almost all the time. - // It is important to craft the code accordingly so that the compiler can use this - // information. (This could also be solved with profile-guided optimization.) - if(simdjson_unlikely(!tape.is_double())) { // branch rarely taken - if(tape.is_uint64()) { - return double(tape.next_tape_value()); - } else if(tape.is_int64()) { - return double(tape.next_tape_value()); - } - return INCORRECT_TYPE; - } - // this is common: - return tape.next_tape_value(); -} -inline simdjson_result element::get_array() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - switch (tape.tape_ref_type()) { - case internal::tape_type::START_ARRAY: - return array(tape); - default: - return INCORRECT_TYPE; - } -} -inline simdjson_result element::get_object() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - switch (tape.tape_ref_type()) { - case internal::tape_type::START_OBJECT: - return object(tape); - default: - return INCORRECT_TYPE; - } -} - -template -simdjson_warn_unused simdjson_inline error_code element::get(T &value) const noexcept { - return get().get(value); -} -// An element-specific version prevents recursion with simdjson_result::get(value) -template<> -simdjson_warn_unused simdjson_inline error_code element::get(element &value) const noexcept { - value = element(tape); - return SUCCESS; -} -template -inline void element::tie(T &value, error_code &error) && noexcept { - error = get(value); -} - -template -simdjson_inline bool element::is() const noexcept { - auto result = get(); - return !result.error(); -} - -template<> inline simdjson_result element::get() const noexcept { return get_array(); } -template<> inline simdjson_result element::get() const noexcept { return get_object(); } -template<> inline simdjson_result element::get() const noexcept { return get_c_str(); } -template<> inline simdjson_result element::get() const noexcept { return get_string(); } -#if SIMDJSON_SUPPORTS_CHAR8_T -template<> inline simdjson_result element::get() const noexcept { return get_u8string(); } -#endif -template<> inline simdjson_result element::get() const noexcept { return get_int64(); } -template<> inline simdjson_result element::get() const noexcept { return get_uint64(); } -template<> inline simdjson_result element::get() const noexcept { return get_double(); } -template<> inline simdjson_result element::get() const noexcept { return get_bool(); } - -inline bool element::is_array() const noexcept { return is(); } -inline bool element::is_object() const noexcept { return is(); } -inline bool element::is_string() const noexcept { return is(); } -inline bool element::is_int64() const noexcept { return is(); } -inline bool element::is_uint64() const noexcept { return is(); } -inline bool element::is_double() const noexcept { return is(); } -inline bool element::is_bool() const noexcept { return is(); } -inline bool element::is_number() const noexcept { return is_int64() || is_uint64() || is_double(); } - -inline bool element::is_null() const noexcept { - return tape.is_null_on_tape(); -} - -inline bool element::is_bigint() const noexcept { - return tape.tape_ref_type() == internal::tape_type::BIGINT; -} - -#if SIMDJSON_EXCEPTIONS - -inline element::operator bool() const noexcept(false) { return get(); } -inline element::operator const char*() const noexcept(false) { return get(); } -inline element::operator std::string_view() const noexcept(false) { return get(); } -inline element::operator uint64_t() const noexcept(false) { return get(); } -inline element::operator int64_t() const noexcept(false) { return get(); } -inline element::operator double() const noexcept(false) { return get(); } -inline element::operator array() const noexcept(false) { return get(); } -inline element::operator object() const noexcept(false) { return get(); } - -inline array::iterator element::begin() const noexcept(false) { - return get().begin(); -} -inline array::iterator element::end() const noexcept(false) { - return get().end(); -} - -#endif // SIMDJSON_EXCEPTIONS - -inline simdjson_result element::operator[](std::string_view key) const noexcept { - return at_key(key); -} -inline simdjson_result element::operator[](const char *key) const noexcept { - return at_key(key); -} - -inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept { - if (simdjson_unlikely(json_pointer[0] != '/')) { - return false; - } - size_t escape = json_pointer.find('~'); - if (escape == std::string_view::npos) { - return true; - } - if (escape == json_pointer.size() - 1) { - return false; - } - if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') { - return false; - } - return true; -} - -inline simdjson_result element::at_pointer(std::string_view json_pointer) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - switch (tape.tape_ref_type()) { - case internal::tape_type::START_OBJECT: - return object(tape).at_pointer(json_pointer); - case internal::tape_type::START_ARRAY: - return array(tape).at_pointer(json_pointer); - default: { - if (!json_pointer.empty()) { // a non-empty string can be invalid, or accessing a primitive (issue 2154) - if (is_pointer_well_formed(json_pointer)) { - return NO_SUCH_FIELD; - } - return INVALID_JSON_POINTER; - } - // an empty string means that we return the current node - dom::element copy(*this); - return simdjson_result(std::move(copy)); - } - } -} - -inline simdjson_result> element::at_path_with_wildcard(std::string_view json_path) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - - const internal::wildcard_json_path path = internal::parse_wildcard_json_path(json_path); - // A path that cannot be parsed matches nothing, whatever the document: say - // so rather than letting the walk skip it as a mismatch below a wildcard. - if (path.tail == internal::wildcard_json_path::tail_type::invalid) { - return INVALID_JSON_POINTER; - } - switch (tape.tape_ref_type()) { - case internal::tape_type::START_OBJECT: - case internal::tape_type::START_ARRAY: - return internal::at_wildcard_json_path(*this, path, 0); - default: - return std::vector{}; - } -} - -inline simdjson_result element::at_path(std::string_view json_path) const noexcept { - auto json_pointer = json_path_to_pointer_conversion(json_path); - if (json_pointer == "-1") { return INVALID_JSON_POINTER; } - return at_pointer(json_pointer); -} -#ifndef SIMDJSON_DISABLE_DEPRECATED_API -[[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] -inline simdjson_result element::at(std::string_view json_pointer) const noexcept { - // version 0.4 of simdjson allowed non-compliant pointers - auto std_pointer = (json_pointer.empty() ? "" : "/") + std::string(json_pointer.begin(), json_pointer.end()); - return at_pointer(std_pointer); -} -#endif // SIMDJSON_DISABLE_DEPRECATED_API - -inline simdjson_result element::at(size_t index) const noexcept { - return get().at(index); -} -inline simdjson_result element::at_key(std::string_view key) const noexcept { - return get().at_key(key); -} -inline simdjson_result element::at_key_case_insensitive(std::string_view key) const noexcept { - return get().at_key_case_insensitive(key); -} -inline bool element::operator<(const element &other) const noexcept { - return tape.json_index < other.tape.json_index; -} -inline bool element::operator==(const element &other) const noexcept { - return tape.json_index == other.tape.json_index; -} - -inline bool element::dump_raw_tape(std::ostream &out) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return tape.doc->dump_raw_tape(out); -} - - -inline std::ostream& operator<<(std::ostream& out, element_type type) { - switch (type) { - case element_type::ARRAY: - return out << "array"; - case element_type::OBJECT: - return out << "object"; - case element_type::INT64: - return out << "int64_t"; - case element_type::UINT64: - return out << "uint64_t"; - case element_type::DOUBLE: - return out << "double"; - case element_type::STRING: - return out << "string"; - case element_type::BOOL: - return out << "bool"; - case element_type::NULL_VALUE: - return out << "null"; - case element_type::BIGINT: - return out << "bigint"; - default: - return out << "unexpected content!!!"; // abort() usage is forbidden in the library - } -} - -} // namespace dom - -} // namespace simdjson - -#endif // SIMDJSON_ELEMENT_INL_H -/* end file simdjson/dom/element-inl.h */ - -#if SIMDJSON_SUPPORTS_RANGES -static_assert(std::ranges::view); -static_assert(std::ranges::sized_range); -#if SIMDJSON_EXCEPTIONS -static_assert(std::ranges::view>); -static_assert(std::ranges::sized_range>); -#endif // SIMDJSON_EXCEPTIONS -#endif // SIMDJSON_SUPPORTS_RANGES - -#endif // SIMDJSON_ARRAY_INL_H -/* end file simdjson/dom/array-inl.h */ -/* including simdjson/dom/document_stream-inl.h: #include "simdjson/dom/document_stream-inl.h" */ -/* begin file simdjson/dom/document_stream-inl.h */ -#ifndef SIMDJSON_DOCUMENT_STREAM_INL_H -#define SIMDJSON_DOCUMENT_STREAM_INL_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/document_stream.h" */ -/* skipped duplicate #include "simdjson/dom/element-inl.h" */ -/* including simdjson/dom/parser-inl.h: #include "simdjson/dom/parser-inl.h" */ -/* begin file simdjson/dom/parser-inl.h */ -#ifndef SIMDJSON_PARSER_INL_H -#define SIMDJSON_PARSER_INL_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/document_stream.h" */ -/* skipped duplicate #include "simdjson/implementation.h" */ -/* skipped duplicate #include "simdjson/internal/dom_parser_implementation.h" */ - -/* skipped duplicate #include "simdjson/error-inl.h" */ -/* skipped duplicate #include "simdjson/padded_string-inl.h" */ -/* skipped duplicate #include "simdjson/dom/document_stream-inl.h" */ -/* skipped duplicate #include "simdjson/dom/element-inl.h" */ - -#include -#include /* memcmp */ - -namespace simdjson { -namespace dom { - -// -// parser inline implementation -// -simdjson_inline parser::parser(size_t max_capacity) noexcept - : _max_capacity{max_capacity}, - loaded_bytes(nullptr) { -} -simdjson_inline parser::parser(parser &&other) noexcept = default; -simdjson_inline parser &parser::operator=(parser &&other) noexcept = default; - -inline bool parser::is_valid() const noexcept { return valid; } -inline int parser::get_error_code() const noexcept { return error; } -inline std::string parser::get_error_message() const noexcept { return error_message(error); } - -inline bool parser::dump_raw_tape(std::ostream &os) const noexcept { - return valid ? doc.dump_raw_tape(os) : false; -} - -inline simdjson_result parser::read_file(std::string_view path) noexcept { - const std::string path_copy(path); - // Open the file - SIMDJSON_PUSH_DISABLE_WARNINGS - SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe - std::FILE *fp = std::fopen(path_copy.c_str(), "rb"); - SIMDJSON_POP_DISABLE_WARNINGS - - if (fp == nullptr) { - return IO_ERROR; - } - - // Get the file size - int ret; -#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS - ret = _fseeki64(fp, 0, SEEK_END); -#else - ret = std::fseek(fp, 0, SEEK_END); -#endif // _WIN64 - if(ret < 0) { - std::fclose(fp); - return IO_ERROR; - } -#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS - __int64 len = _ftelli64(fp); - if(len == -1L) { - std::fclose(fp); - return IO_ERROR; - } -#else - long len = std::ftell(fp); - if((len < 0) || (len == LONG_MAX)) { - std::fclose(fp); - return IO_ERROR; - } -#endif - - // Make sure we have enough capacity to load the file - if (_loaded_bytes_capacity < size_t(len)) { - loaded_bytes.reset( internal::allocate_padded_buffer(len) ); - if (!loaded_bytes) { - std::fclose(fp); - return MEMALLOC; - } - _loaded_bytes_capacity = len; - } - - // Read the string - std::rewind(fp); - size_t bytes_read = std::fread(loaded_bytes.get(), 1, len, fp); - if (std::fclose(fp) != 0 || bytes_read != size_t(len)) { - return IO_ERROR; - } - - return bytes_read; -} - -inline simdjson_result parser::load(std::string_view path) & noexcept { - return load_into_document(doc, path); -} - -inline simdjson_result parser::load_into_document(document& provided_doc, std::string_view path) & noexcept { - size_t len; - auto _error = read_file(path).get(len); - if (_error) { return _error; } - return parse_into_document(provided_doc, loaded_bytes.get(), len, false); -} - -inline simdjson_result parser::load_many(std::string_view path, size_t batch_size) noexcept { - size_t len; - auto _error = read_file(path).get(len); - if (_error) { return _error; } - if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } - return document_stream(*this, reinterpret_cast(loaded_bytes.get()), len, batch_size); -} - -inline simdjson_result parser::parse_into_document(document& provided_doc, const uint8_t *buf, size_t len, bool realloc_if_needed) & noexcept { - // Important: we need to ensure that document has enough capacity. - // Important: It is possible that provided_doc is actually the internal 'doc' within the parser!!! - error_code _error = ensure_capacity(provided_doc, len); - if (_error) { return _error; } - if (realloc_if_needed) { - // Make sure we have enough capacity to copy len bytes - if (!loaded_bytes || _loaded_bytes_capacity < len) { - loaded_bytes.reset( internal::allocate_padded_buffer(len) ); - if (!loaded_bytes) { - return MEMALLOC; - } - _loaded_bytes_capacity = len; - } - std::memcpy(static_cast(loaded_bytes.get()), buf, len); - buf = reinterpret_cast(loaded_bytes.get()); - } - - if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { - buf += 3; - len -= 3; - } - implementation->_number_as_string = _number_as_string; - _error = implementation->parse(buf, len, provided_doc); - - if (_error) { return _error; } - - return provided_doc.root(); -} - -inline simdjson_result parser::parse_into_document_unpadded(document& provided_doc, const uint8_t *buf, size_t len) & noexcept { - // Like parse_into_document with realloc_if_needed=false (no copy, parse in - // place), but we tell the implementation the buffer is not padded so neither - // stage reads past buf+len: stage 1 copies the last block instead of loading - // it in place, and in stage 2 string unescaping is bounded, near-the-end - // numbers are parsed from a padded copy, and atoms use length-aware - // validators (see tape_builder). - error_code _error = ensure_capacity(provided_doc, len); - if (_error) { return _error; } - - if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { - buf += 3; - len -= 3; - } - implementation->_number_as_string = _number_as_string; - implementation->_unpadded = true; - _error = implementation->parse(buf, len, provided_doc); - implementation->_unpadded = false; // restore so later padded parses use the fast path - - if (_error) { return _error; } - - return provided_doc.root(); -} - -simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const char *buf, size_t len, bool realloc_if_needed) & noexcept { - return parse_into_document(provided_doc, reinterpret_cast(buf), len, realloc_if_needed); -} -simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const std::string &s) & noexcept { - return parse_into_document(provided_doc, s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); -} -simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const padded_string &s) & noexcept { - return parse_into_document(provided_doc, s.data(), s.length(), false); -} - - -inline simdjson_result parser::parse(const uint8_t *buf, size_t len, bool realloc_if_needed) & noexcept { - return parse_into_document(doc, buf, len, realloc_if_needed); -} - -simdjson_inline simdjson_result parser::parse(const char *buf, size_t len, bool realloc_if_needed) & noexcept { - return parse(reinterpret_cast(buf), len, realloc_if_needed); -} -simdjson_inline simdjson_result parser::parse(const std::string &s) & noexcept { - return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); -} -simdjson_inline simdjson_result parser::parse(const padded_string &s) & noexcept { - return parse(s.data(), s.length(), false); -} -simdjson_inline simdjson_result parser::parse(const padded_string_view &v) & noexcept { - if (!v.has_sufficient_padding()) { return INSUFFICIENT_PADDING; } - padded_string_view json(v); - json.use_trailing_whitespace_as_padding(); - return parse(json.data(), json.length(), false); -} - -inline simdjson_result parser::parse_unpadded(const uint8_t *buf, size_t len) & noexcept { - return parse_into_document_unpadded(doc, buf, len); -} -simdjson_inline simdjson_result parser::parse_unpadded(const char *buf, size_t len) & noexcept { - return parse_unpadded(reinterpret_cast(buf), len); -} -simdjson_inline simdjson_result parser::parse_unpadded(std::string_view s) & noexcept { - return parse_unpadded(reinterpret_cast(s.data()), s.size()); -} - -inline simdjson_result parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept { - return parse_many(buf, len, batch_size, stream_format::whitespace_delimited); -} -inline simdjson_result parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept { - return parse_many(reinterpret_cast(buf), len, batch_size); -} -inline simdjson_result parser::parse_many(const std::string &s, size_t batch_size) noexcept { - return parse_many(padded_string_view(s), batch_size); -} -inline simdjson_result parser::parse_many(const padded_string &s, size_t batch_size) noexcept { - return parse_many(s.data(), s.length(), batch_size); -} -inline simdjson_result parser::parse_many(const padded_string_view &v, size_t batch_size) noexcept { - if (!v.has_sufficient_padding()) { return INSUFFICIENT_PADDING; } - padded_string_view json(v); - json.use_trailing_whitespace_as_padding(); - return parse_many(json.data(), json.length(), batch_size); -} - -inline simdjson_result parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) noexcept { - if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } - if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { - buf += 3; - len -= 3; - } - if (format == stream_format::comma_delimited_array) { - // Strip leading JSON whitespace. - while (len > 0 && (buf[0] == ' ' || buf[0] == '\t' || buf[0] == '\n' || buf[0] == '\r')) { - buf++; len--; - } - // Expect the opening '['. - if (len == 0 || buf[0] != '[') { return TAPE_ERROR; } - buf++; len--; - // Strip trailing JSON whitespace. - while (len > 0 && (buf[len-1] == ' ' || buf[len-1] == '\t' || buf[len-1] == '\n' || buf[len-1] == '\r')) { - len--; - } - // Expect the closing ']'. - if (len == 0 || buf[len-1] != ']') { return TAPE_ERROR; } - len--; - // Fall through to comma_delimited over the array contents. - format = stream_format::comma_delimited; - } - return document_stream(*this, buf, len, batch_size, format); -} -inline simdjson_result parser::parse_many(const char *buf, size_t len, size_t batch_size, stream_format format) noexcept { - return parse_many(reinterpret_cast(buf), len, batch_size, format); -} -inline simdjson_result parser::parse_many(const std::string &s, size_t batch_size, stream_format format) noexcept { - return parse_many(padded_string_view(s), batch_size, format); -} -inline simdjson_result parser::parse_many(const padded_string &s, size_t batch_size, stream_format format) noexcept { - return parse_many(s.data(), s.length(), batch_size, format); -} -inline simdjson_result parser::parse_many(const padded_string_view &v, size_t batch_size, stream_format format) noexcept { - if (!v.has_sufficient_padding()) { return INSUFFICIENT_PADDING; } - padded_string_view json(v); - json.use_trailing_whitespace_as_padding(); - return parse_many(json.data(), json.length(), batch_size, format); -} - -simdjson_inline size_t parser::capacity() const noexcept { - return implementation ? implementation->capacity() : 0; -} -simdjson_inline size_t parser::max_capacity() const noexcept { - return _max_capacity; -} -simdjson_pure simdjson_inline size_t parser::max_depth() const noexcept { - return implementation ? implementation->max_depth() : DEFAULT_MAX_DEPTH; -} - -simdjson_warn_unused -inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept { - // - // Reallocate implementation if needed - // - error_code err; - if (implementation) { - err = implementation->allocate(capacity, max_depth); - } else { - err = simdjson::get_active_implementation()->create_dom_parser_implementation(capacity, max_depth, implementation); - } - if (err) { return err; } - return SUCCESS; -} - -#ifndef SIMDJSON_DISABLE_DEPRECATED_API -simdjson_warn_unused -inline bool parser::allocate_capacity(size_t capacity, size_t max_depth) noexcept { - return !allocate(capacity, max_depth); -} -#endif // SIMDJSON_DISABLE_DEPRECATED_API - -inline error_code parser::ensure_capacity(size_t desired_capacity) noexcept { - return ensure_capacity(doc, desired_capacity); -} - - -inline error_code parser::ensure_capacity(document& target_document, size_t desired_capacity) noexcept { - // 1. It is wasteful to allocate a document and a parser for documents spanning less than MINIMAL_DOCUMENT_CAPACITY bytes. - // 2. If we allow desired_capacity = 0 then it is possible to exit this function with implementation == nullptr. - if(desired_capacity < MINIMAL_DOCUMENT_CAPACITY) { desired_capacity = MINIMAL_DOCUMENT_CAPACITY; } - // If we don't have enough capacity, (try to) automatically bump it. - // If the document needs allocation, do it too. - // Both in one if statement to minimize unlikely branching. - // - // Note: we must make sure that this function is called if capacity() == 0. We do so because we - // ensure that desired_capacity > 0. - if (simdjson_unlikely(capacity() < desired_capacity || target_document.capacity() < desired_capacity)) { - if (desired_capacity > max_capacity()) { - return error = CAPACITY; - } - error_code err1 = target_document.capacity() < desired_capacity ? target_document.allocate(desired_capacity) : SUCCESS; - error_code err2 = capacity() < desired_capacity ? allocate(desired_capacity, max_depth()) : SUCCESS; - if(err1 != SUCCESS) { return error = err1; } - if(err2 != SUCCESS) { return error = err2; } - } - return SUCCESS; -} - -simdjson_inline void parser::set_max_capacity(size_t max_capacity) noexcept { - if(max_capacity > MINIMAL_DOCUMENT_CAPACITY) { - _max_capacity = max_capacity; - } else { - _max_capacity = MINIMAL_DOCUMENT_CAPACITY; - } -} - -} // namespace dom -} // namespace simdjson - -#endif // SIMDJSON_PARSER_INL_H -/* end file simdjson/dom/parser-inl.h */ -/* skipped duplicate #include "simdjson/error-inl.h" */ -/* skipped duplicate #include "simdjson/internal/dom_parser_implementation.h" */ - -namespace simdjson { -namespace dom { - -#ifdef SIMDJSON_THREADS_ENABLED - -inline void stage1_worker::finish() { - // After calling "run" someone would call finish() to wait - // for the end of the processing. - // This function will wait until either the thread has done - // the processing or, else, the destructor has been called. - std::unique_lock lock(locking_mutex); - cond_var.wait(lock, [this]{return has_work == false;}); -} - -inline stage1_worker::~stage1_worker() { - // The thread may never outlive the stage1_worker instance - // and will always be stopped/joined before the stage1_worker - // instance is gone. - stop_thread(); -} - -inline void stage1_worker::start_thread() { - std::unique_lock lock(locking_mutex); - if(thread.joinable()) { - return; // This should never happen but we never want to create more than one thread. - } - thread = std::thread([this]{ - while(true) { - std::unique_lock thread_lock(locking_mutex); - // We wait for either "run" or "stop_thread" to be called. - cond_var.wait(thread_lock, [this]{return has_work || !can_work;}); - // If, for some reason, the stop_thread() method was called (i.e., the - // destructor of stage1_worker is called, then we want to immediately destroy - // the thread (and not do any more processing). - if(!can_work) { - break; - } - this->owner->stage1_thread_error = this->owner->run_stage1(*this->stage1_thread_parser, - this->_next_batch_start); - this->has_work = false; - // The condition variable call should be moved after thread_lock.unlock() for performance - // reasons but thread sanitizers may report it as a data race if we do. - // See https://stackoverflow.com/questions/35775501/c-should-condition-variable-be-notified-under-lock - cond_var.notify_one(); // will notify "finish" - thread_lock.unlock(); - } - } - ); -} - - -inline void stage1_worker::stop_thread() { - std::unique_lock lock(locking_mutex); - // We have to make sure that all locks can be released. - can_work = false; - has_work = false; - cond_var.notify_all(); - lock.unlock(); - if(thread.joinable()) { - thread.join(); - } -} - -inline void stage1_worker::run(document_stream * ds, dom::parser * stage1, size_t next_batch_start) { - std::unique_lock lock(locking_mutex); - owner = ds; - _next_batch_start = next_batch_start; - stage1_thread_parser = stage1; - has_work = true; - // The condition variable call should be moved after thread_lock.unlock() for performance - // reasons but thread sanitizers may report it as a data race if we do. - // See https://stackoverflow.com/questions/35775501/c-should-condition-variable-be-notified-under-lock - cond_var.notify_one(); // will notify the thread lock that we have work - lock.unlock(); -} -#endif - -simdjson_inline document_stream::document_stream( - dom::parser &_parser, - const uint8_t *_buf, - size_t _len, - size_t _batch_size, - stream_format _format -) noexcept - : parser{&_parser}, - buf{_buf}, - len{_len}, - batch_size{_batch_size <= MINIMAL_BATCH_SIZE ? MINIMAL_BATCH_SIZE : _batch_size}, - format{_format}, - error{SUCCESS} -#ifdef SIMDJSON_THREADS_ENABLED - , use_thread(_parser.threaded) // we need to make a copy because _parser.threaded can change -#endif -{ -#ifdef SIMDJSON_THREADS_ENABLED - if(worker.get() == nullptr) { - error = MEMALLOC; - } -#endif -} - -simdjson_inline document_stream::document_stream() noexcept - : parser{nullptr}, - buf{nullptr}, - len{0}, - batch_size{0}, - format{stream_format::whitespace_delimited}, - error{UNINITIALIZED} -#ifdef SIMDJSON_THREADS_ENABLED - , use_thread(false) -#endif -{ -} - -simdjson_inline document_stream::~document_stream() noexcept { -#ifdef SIMDJSON_THREADS_ENABLED - worker.reset(); -#endif -} - -simdjson_inline document_stream::iterator::iterator() noexcept - : stream{nullptr}, finished{true} { -} - -simdjson_inline document_stream::iterator document_stream::begin() noexcept { - start(); - // If there are no documents, we're finished. - return iterator(this, error == EMPTY); -} - -simdjson_inline document_stream::iterator document_stream::end() noexcept { - return iterator(this, true); -} - -simdjson_inline document_stream::iterator::iterator(document_stream* _stream, bool is_end) noexcept - : stream{_stream}, finished{is_end} { -} - -simdjson_inline document_stream::iterator::reference document_stream::iterator::operator*() noexcept { - // Note that in case of error, we do not yet mark - // the iterator as "finished": this detection is done - // in the operator++ function since it is possible - // to call operator++ repeatedly while omitting - // calls to operator*. - if (stream->error) { return stream->error; } - return stream->parser->doc.root(); -} - -simdjson_inline document_stream::iterator& document_stream::iterator::operator++() noexcept { - // If there is an error, then we want the iterator - // to be finished, no matter what. (E.g., we do not - // keep generating documents with errors, or go beyond - // a document with errors.) - // - // Users do not have to call "operator*()" when they use operator++, - // so we need to end the stream in the operator++ function. - // - // Note that setting finished = true is essential otherwise - // we would enter an infinite loop. - if (stream->error) { finished = true; } - // Note that stream->error() is guarded against error conditions - // (it will immediately return if stream->error casts to false). - // In effect, this next function does nothing when (stream->error) - // is true (hence the risk of an infinite loop). - stream->next(); - // If that was the last document, we're finished. - // It is the only type of error we do not want to appear - // in operator*. - if (stream->error == EMPTY) { finished = true; } - // If we had any other kind of error (not EMPTY) then we want - // to pass it along to the operator* and we cannot mark the result - // as "finished" just yet. - return *this; -} - -simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { - return finished != other.finished; -} - -inline void document_stream::start() noexcept { - if (error) { return; } - error = parser->ensure_capacity(batch_size); - if (error) { return; } - parser->implementation->_number_as_string = parser->number_as_string(); - // Always run the first stage 1 parse immediately - batch_start = 0; - error = run_stage1(*parser, batch_start); - while(error == EMPTY) { - // In exceptional cases, we may start with an empty block - batch_start = next_batch_start(); - if (batch_start >= len) { return; } - error = run_stage1(*parser, batch_start); - } - if (error) { return; } - load_stage2(); -#ifdef SIMDJSON_THREADS_ENABLED - if (use_thread && next_batch_start() < len) { - // Kick off the first thread if needed - // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which - // then parses every other batch: give it the same settings. - stage1_thread_parser.set_max_capacity(parser->max_capacity()); - stage1_thread_parser.number_as_string(parser->number_as_string()); - error = stage1_thread_parser.allocate(batch_size, parser->max_depth()); - if (error) { return; } - error = stage1_thread_parser.ensure_capacity(batch_size); - if (error) { return; } - stage1_thread_parser.implementation->_number_as_string = parser->number_as_string(); - worker->start_thread(); - start_stage1_thread(); - if (error) { return; } - } -#endif // SIMDJSON_THREADS_ENABLED - next(); -} - -simdjson_inline size_t document_stream::iterator::current_index() const noexcept { - return stream->doc_index; -} - -simdjson_inline std::string_view document_stream::iterator::source() const noexcept { - const char* start = reinterpret_cast(stream->buf) + current_index(); - if (stream->error) { - return std::string_view(start, stream->len - current_index()); - } - bool object_or_array = ((*start == '[') || (*start == '{')); - if(object_or_array) { - size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index - 1]; - return std::string_view(start, next_doc_index - current_index() + 1); - } else { - size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index]; - size_t svlen = next_doc_index - current_index(); - // When the scalar is followed by a truncated document, the structural - // indexes of that document were dropped and next_doc_index is the end of - // the input, so we bound the scalar by scanning the token itself. - size_t token_len = 0; - if (*start == '"') { - token_len = 1; - while (token_len < svlen) { - char c = start[token_len++]; - if (c == '\\') { - token_len++; - } else if (c == '"') { - break; - } - } - } else { - while (token_len < svlen) { - char c = start[token_len]; - if (std::isspace(static_cast(c)) || c == ',' || c == '{' || c == '[' || c == '\0' || static_cast(c) == 0x1E) { - break; - } - token_len++; - } - } - if (token_len > 0 && token_len < svlen) { - svlen = token_len; - } - // Trim trailing whitespace, NUL, and RS (0x1E). In RFC 7464 json_sequence - // mode the scanner classifies RS as a scalar character, so an RS-prefixed - // scalar document (number/true/false/null/string) has no closing structural - // index and the slice runs all the way up to the next document's RS. RS - // cannot legally appear in a JSON value at the source level (control - // characters in strings must be escaped as \u001E), so stripping it is - // safe in every stream_format. - while(svlen > 1 && (std::isspace(static_cast(start[svlen-1])) || start[svlen-1] == '\0' || static_cast(start[svlen-1]) == 0x1E || (stream->format == stream_format::comma_delimited && start[svlen-1] == ','))) { - svlen--; - } - return std::string_view(start, svlen); - } -} - - -inline void document_stream::next() noexcept { - // We always exit at once, once in an error condition. - if (error) { return; } - - // Load the next document from the batch - doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index]; - error = stage2_next_fn(*parser->implementation, parser->doc); - // If that was the last document in the batch, load another batch (if available) - while (error == EMPTY) { - batch_start = next_batch_start(); - if (batch_start >= len) { break; } - -#ifdef SIMDJSON_THREADS_ENABLED - if(use_thread) { - load_from_stage1_thread(); - } else { - error = run_stage1(*parser, batch_start); - } -#else - error = run_stage1(*parser, batch_start); -#endif - if (error) { continue; } // If the error was EMPTY, we may want to load another batch. - load_stage2(); - // Run stage 2 on the first document in the batch - doc_index = batch_start + parser->implementation->structural_indexes[parser->implementation->next_structural_index]; - error = stage2_next_fn(*parser->implementation, parser->doc); - } -} -inline size_t document_stream::size_in_bytes() const noexcept { - return len; -} - -inline size_t document_stream::truncated_bytes() const noexcept { - // Stage 1 returns EMPTY on zero-length input before it writes the index - // sentinels read below, so they would still hold a previous stream's values. - if (len == 0) { return 0; } - if(error == CAPACITY) { return len - batch_start; } - return parser->implementation->structural_indexes[parser->implementation->n_structural_indexes] - parser->implementation->structural_indexes[parser->implementation->n_structural_indexes + 1]; -} - -inline size_t document_stream::next_batch_start() const noexcept { - return batch_start + parser->implementation->structural_indexes[parser->implementation->n_structural_indexes]; -} - -// Stage 1 cuts the batch after the last document that looks complete, and -// leaves a sentinel structural index past it: stage 2 validates the documents, -// and stops an invalid one at the sentinel because the byte there cannot -// continue a document. That byte is not part of the batch, though: it is the -// start of the next document, padding (which the caller need not initialize) -// or the `]` that comma_delimited_array strips. When it is ',', ':', ']' or -// '}', or when the batch ends where a value is expected, an invalid document -// such as `[2` (before a truncated `[["`) could be accepted and run past the -// batch, into stale indexes such as the quote of an unclosed string, and so -// past the buffer. Stage 2 then walks this batch with an explicit bound (see -// load_stage2). Other batches keep the unchecked walk. -// -// set_bounded_batch() makes that decision for the batch stage 1 just indexed. -// Out of line: it runs once per batch, and inlined it would weigh on next(). -simdjson_never_inline void set_bounded_batch(internal::dom_parser_implementation &impl, const uint8_t *batch) noexcept { - bool bounded = false; - const uint32_t n = impl.n_structural_indexes; - if (n > 0) { - const uint8_t next = batch[impl.structural_indexes[n]]; - const uint8_t last = batch[impl.structural_indexes[n - 1]]; - bounded = next == ',' || next == ':' || next == ']' || next == '}' || - last == ',' || last == ':' || last == '[' || last == '{'; - } - impl._bounded_batch = bounded; -} - -inline void document_stream::load_stage2() noexcept { - stage2_next_fn = parser->implementation->stage2_next_function(parser->implementation->_bounded_batch); -} - -inline error_code document_stream::run_stage1(dom::parser &p, size_t _batch_start) noexcept { - size_t remaining = len - _batch_start; - stage1_mode mode; - size_t batch_len; - if (remaining <= batch_size) { - // Final batch - switch (format) { - case stream_format::json_sequence: - mode = stage1_mode::json_sequence_final; - break; - case stream_format::comma_delimited: - mode = stage1_mode::comma_delimited_final; - break; - default: - mode = stage1_mode::streaming_final; - break; - } - batch_len = remaining; - } else { - // Partial batch - switch (format) { - case stream_format::json_sequence: - mode = stage1_mode::json_sequence_partial; - break; - case stream_format::comma_delimited: - mode = stage1_mode::comma_delimited_partial; - break; - default: - mode = stage1_mode::streaming_partial; - break; - } - batch_len = batch_size; - } - error_code err = p.implementation->stage1(&buf[_batch_start], batch_len, mode); - if (!err) { set_bounded_batch(*p.implementation, &buf[_batch_start]); } - return err; -} - -#ifdef SIMDJSON_THREADS_ENABLED - -inline void document_stream::load_from_stage1_thread() noexcept { - worker->finish(); - // Swap to the parser that was loaded up in the thread. Make sure the parser has - // enough memory to swap to, as well. - std::swap(*parser, stage1_thread_parser); - error = stage1_thread_error; - // An EMPTY batch (no complete document in it, e.g. a run of white space at - // least batch_size long) is not the end of the stream: next() moves on to - // the batch after it, so queue that batch now as we would after a success. - // Returning here would leave no job for the next call, which then swaps - // the stale previous batch back in and ends the stream early. - if (error && error != EMPTY) { return; } - - // If there's anything left, start the stage 1 thread! - if (next_batch_start() < len) { - start_stage1_thread(); - } -} - -inline void document_stream::start_stage1_thread() noexcept { - // we call the thread on a lambda that will update - // this->stage1_thread_error - // there is only one thread that may write to this value - // TODO this is NOT exception-safe. - this->stage1_thread_error = UNINITIALIZED; // In case something goes wrong, make sure it's an error - size_t _next_batch_start = this->next_batch_start(); - - worker->run(this, & this->stage1_thread_parser, _next_batch_start); -} - -#endif // SIMDJSON_THREADS_ENABLED - -} // namespace dom - -simdjson_inline simdjson_result::simdjson_result() noexcept - : simdjson_result_base() { -} -simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept - : simdjson_result_base(error) { -} -simdjson_inline simdjson_result::simdjson_result(dom::document_stream &&value) noexcept - : simdjson_result_base(std::forward(value)) { -} - -#if SIMDJSON_EXCEPTIONS -simdjson_inline dom::document_stream::iterator simdjson_result::begin() noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.begin(); -} -simdjson_inline dom::document_stream::iterator simdjson_result::end() noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.end(); -} -#else // SIMDJSON_EXCEPTIONS -#ifndef SIMDJSON_DISABLE_DEPRECATED_API -simdjson_inline dom::document_stream::iterator simdjson_result::begin() noexcept { - first.error = error(); - return first.begin(); -} -simdjson_inline dom::document_stream::iterator simdjson_result::end() noexcept { - first.error = error(); - return first.end(); -} -#endif // SIMDJSON_DISABLE_DEPRECATED_API -#endif // SIMDJSON_EXCEPTIONS - -} // namespace simdjson -#endif // SIMDJSON_DOCUMENT_STREAM_INL_H -/* end file simdjson/dom/document_stream-inl.h */ -/* including simdjson/dom/document-inl.h: #include "simdjson/dom/document-inl.h" */ -/* begin file simdjson/dom/document-inl.h */ -#ifndef SIMDJSON_DOCUMENT_INL_H -#define SIMDJSON_DOCUMENT_INL_H - -// Inline implementations go in here. - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/document.h" */ -/* skipped duplicate #include "simdjson/dom/element-inl.h" */ -/* skipped duplicate #include "simdjson/internal/tape_ref-inl.h" */ -/* including simdjson/internal/jsonformatutils.h: #include "simdjson/internal/jsonformatutils.h" */ -/* begin file simdjson/internal/jsonformatutils.h */ -#ifndef SIMDJSON_INTERNAL_JSONFORMATUTILS_H -#define SIMDJSON_INTERNAL_JSONFORMATUTILS_H - -/* skipped duplicate #include "simdjson/base.h" */ -#include -#include -#include - -namespace simdjson { -namespace internal { - -inline std::ostream& operator<<(std::ostream& out, const escape_json_string &str); - -class escape_json_string { -public: - escape_json_string(std::string_view _str) noexcept : str{_str} {} - operator std::string() const noexcept { std::stringstream s; s << *this; return s.str(); } -private: - std::string_view str; - friend std::ostream& operator<<(std::ostream& out, const escape_json_string &unescaped); -}; - -inline std::ostream& operator<<(std::ostream& out, const escape_json_string &unescaped) { - for (size_t i=0; i(unescaped.str[i]) <= 0x1F) { - // TODO can this be done once at the beginning, or will it mess up << char? - std::ios::fmtflags f(out.flags()); - out << "\\u" << std::hex << std::setw(4) << std::setfill('0') << int(unescaped.str[i]); - out.flags(f); - } else { - out << unescaped.str[i]; - } - } - } - return out; -} - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_JSONFORMATUTILS_H -/* end file simdjson/internal/jsonformatutils.h */ - -#include - -namespace simdjson { -namespace dom { - -// -// document inline implementation -// -inline element document::root() const noexcept { - return element(internal::tape_ref(this, 1)); -} -simdjson_warn_unused -inline size_t document::capacity() const noexcept { - return allocated_capacity; -} - -simdjson_warn_unused -inline error_code document::allocate(size_t capacity) noexcept { - if (capacity == 0) { - string_buf.reset(); - tape.reset(); - allocated_capacity = 0; - return SUCCESS; - } - if (capacity > SIMDJSON_MAXSIZE_BYTES) { - return CAPACITY; - } - - // a pathological input like "[[[[..." would generate capacity tape elements, so - // need a capacity of at least capacity + 1, but it is also possible to do - // worse with "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6" - //where capacity + 1 tape elements are - // generated, see issue https://github.com/simdjson/simdjson/issues/345 - if(capacity + 3 < capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t tape_capacity = SIMDJSON_ROUNDUP_N(capacity + 3, 64); - // a document with only zero-length strings... could have capacity/3 string - // and we would need capacity/3 * 5 bytes on the string buffer - if(5 * (capacity / 3) + SIMDJSON_PADDING < SIMDJSON_PADDING) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t string_capacity = SIMDJSON_ROUNDUP_N(5 * (capacity / 3) + SIMDJSON_PADDING, 64); - string_buf.reset( new (std::nothrow) uint8_t[string_capacity]); - tape.reset(new (std::nothrow) uint64_t[tape_capacity]); - if(!(string_buf && tape)) { - allocated_capacity = 0; - string_buf.reset(); - tape.reset(); - return MEMALLOC; - } - // Technically the allocated_capacity might be larger than capacity - // so the next line is pessimistic. - allocated_capacity = capacity; - return SUCCESS; -} - -inline bool document::dump_raw_tape(std::ostream &os) const noexcept { - uint32_t string_length; - size_t tape_idx = 0; - uint64_t tape_val = tape[tape_idx]; - uint8_t type = uint8_t(tape_val >> 56); - os << tape_idx << " : " << type; - tape_idx++; - size_t how_many = 0; - if (type == 'r') { - how_many = size_t(tape_val & internal::JSON_VALUE_MASK); - } else { - // Error: no starting root node? - return false; - } - os << "\t// pointing to " << how_many << " (right after last node)\n"; - uint64_t payload; - for (; tape_idx < how_many; tape_idx++) { - os << tape_idx << " : "; - tape_val = tape[tape_idx]; - payload = tape_val & internal::JSON_VALUE_MASK; - type = uint8_t(tape_val >> 56); - switch (type) { - case '"': // we have a string - os << "string \""; - std::memcpy(&string_length, string_buf.get() + payload, sizeof(uint32_t)); - os << internal::escape_json_string(std::string_view( - reinterpret_cast(string_buf.get() + payload + sizeof(uint32_t)), - string_length - )); - os << '"'; - os << '\n'; - break; - case 'l': // we have a long int - if (tape_idx + 1 >= how_many) { - return false; - } - os << "integer " << static_cast(tape[++tape_idx]) << "\n"; - break; - case 'u': // we have a long uint - if (tape_idx + 1 >= how_many) { - return false; - } - os << "unsigned integer " << tape[++tape_idx] << "\n"; - break; - case 'd': // we have a double - os << "float "; - if (tape_idx + 1 >= how_many) { - return false; - } - double answer; - std::memcpy(&answer, &tape[++tape_idx], sizeof(answer)); - os << answer << '\n'; - break; - case 'n': // we have a null - os << "null\n"; - break; - case 't': // we have a true - os << "true\n"; - break; - case 'f': // we have a false - os << "false\n"; - break; - case '{': // we have an object - os << "{\t// pointing to next tape location " << uint32_t(payload) - << " (first node after the scope), " - << " saturated count " - << ((payload >> 32) & internal::JSON_COUNT_MASK)<< "\n"; - break; case '}': // we end an object - os << "}\t// pointing to previous tape location " << uint32_t(payload) - << " (start of the scope)\n"; - break; - case '[': // we start an array - os << "[\t// pointing to next tape location " << uint32_t(payload) - << " (first node after the scope), " - << " saturated count " - << ((payload >> 32) & internal::JSON_COUNT_MASK)<< "\n"; - break; - case ']': // we end an array - os << "]\t// pointing to previous tape location " << uint32_t(payload) - << " (start of the scope)\n"; - break; - case 'r': // we start and end with the root node - // should we be hitting the root node? - return false; - case 'Z': // we have a big integer - os << "bigint "; - std::memcpy(&string_length, string_buf.get() + payload, sizeof(uint32_t)); - os << std::string_view( - reinterpret_cast(string_buf.get() + payload + sizeof(uint32_t)), - string_length - ); - os << '\n'; - break; - default: - return false; - } - } - tape_val = tape[tape_idx]; - payload = tape_val & internal::JSON_VALUE_MASK; - type = uint8_t(tape_val >> 56); - os << tape_idx << " : " << type << "\t// pointing to " << payload - << " (start root)\n"; - return true; -} - -} // namespace dom -} // namespace simdjson - -#endif // SIMDJSON_DOCUMENT_INL_H -/* end file simdjson/dom/document-inl.h */ -/* skipped duplicate #include "simdjson/dom/element-inl.h" */ -/* skipped duplicate #include "simdjson/dom/object-inl.h" */ -/* skipped duplicate #include "simdjson/dom/parser-inl.h" */ -/* skipped duplicate #include "simdjson/internal/tape_ref-inl.h" */ -/* including simdjson/dom/serialization-inl.h: #include "simdjson/dom/serialization-inl.h" */ -/* begin file simdjson/dom/serialization-inl.h */ - -#ifndef SIMDJSON_SERIALIZATION_INL_H -#define SIMDJSON_SERIALIZATION_INL_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/parser.h" */ -/* skipped duplicate #include "simdjson/dom/serialization.h" */ -/* including simdjson/internal/jeaiii.h: #include "simdjson/internal/jeaiii.h" */ -/* begin file simdjson/internal/jeaiii.h */ -#ifndef SIMDJSON_INTERNAL_JEAIII_H -#define SIMDJSON_INTERNAL_JEAIII_H - -/* skipped duplicate #include "simdjson/base.h" */ -#include -#include - -namespace simdjson { -namespace internal { - -// Integer to decimal: James Edward Anhalt III's algorithm -static const char jeaiii_dd[201] = - "00010203040506070809101112131415161718192021222324252627282930313233343536373839" - "40414243444546474849505152535455565758596061626364656667686970717273747576777879" - "8081828384858687888990919293949596979899"; -static const char jeaiii_fd[201] = - "0\0" "1\0" "2\0" "3\0" "4\0" "5\0" "6\0" "7\0" "8\0" "9\0" - "10111213141516171819202122232425262728293031323334353637383940414243444546474849" - "50515253545556575859606162636465666768697071727374757677787980818283848586878889" - "90919293949596979899"; - -simdjson_really_inline void jeaiii_write_dd(char *p, uint64_t k) noexcept { - std::memcpy(p, &jeaiii_dd[2 * k], 2); -} -simdjson_really_inline void jeaiii_write_fd(char *p, uint64_t k) noexcept { - std::memcpy(p, &jeaiii_fd[2 * k], 2); -} - -// Caller guarantees n < 10^8. Writes 1 to 8 digits. -simdjson_really_inline char *jeaiii_lt1e8(char *b, uint32_t n) noexcept { - constexpr uint64_t mask24 = (uint64_t(1) << 24) - 1; - constexpr uint64_t mask32 = (uint64_t(1) << 32) - 1; - if (n < 100) { - jeaiii_write_fd(b, n); - return n < 10 ? b + 1 : b + 2; - } - if (n < 1000000) { - if (n < 10000) { - const uint32_t f0 = uint32_t(10 * (1 << 24) / 1e3 + 1) * n; - jeaiii_write_fd(b, f0 >> 24); - b -= n < 1000; - const uint32_t f2 = uint32_t(f0 & mask24) * 100; - jeaiii_write_dd(b + 2, f2 >> 24); - return b + 4; - } - const uint64_t f0 = uint64_t(10 * (1ull << 32) / 1e5 + 1) * n; - jeaiii_write_fd(b, f0 >> 32); - b -= n < 100000; - const uint64_t f2 = (f0 & mask32) * 100; - jeaiii_write_dd(b + 2, f2 >> 32); - const uint64_t f4 = (f2 & mask32) * 100; - jeaiii_write_dd(b + 4, f4 >> 32); - return b + 6; - } - const uint64_t f0 = uint64_t(10 * (1ull << 48) / 1e7 + 1) * n >> 16; - jeaiii_write_fd(b, f0 >> 32); - b -= n < 10000000; - const uint64_t f2 = (f0 & mask32) * 100; - jeaiii_write_dd(b + 2, f2 >> 32); - const uint64_t f4 = (f2 & mask32) * 100; - jeaiii_write_dd(b + 4, f4 >> 32); - const uint64_t f6 = (f4 & mask32) * 100; - jeaiii_write_dd(b + 6, f6 >> 32); - return b + 8; -} - -// Caller guarantees z < 10^8. Always writes exactly 8 digits. -simdjson_really_inline char *jeaiii_8_digits(char *b, uint32_t z) noexcept { - constexpr uint64_t mask32 = (uint64_t(1) << 32) - 1; - const uint64_t f0 = (uint64_t((1ull << 48) / 1e6 + 1) * z >> 16) + 1; - jeaiii_write_dd(b, f0 >> 32); - const uint64_t f2 = (f0 & mask32) * 100; - jeaiii_write_dd(b + 2, f2 >> 32); - const uint64_t f4 = (f2 & mask32) * 100; - jeaiii_write_dd(b + 4, f4 >> 32); - const uint64_t f6 = (f4 & mask32) * 100; - jeaiii_write_dd(b + 6, f6 >> 32); - return b + 8; -} - -// Caller guarantees 10^8 <= n < 2^32. Writes 9 or 10 digits. -simdjson_really_inline char *jeaiii_9_or_10(char *b, uint64_t n) noexcept { - constexpr uint64_t mask57 = (uint64_t(1) << 57) - 1; - const uint64_t f0 = uint64_t(10 * (1ull << 57) / 1e9 + 1) * n; - jeaiii_write_fd(b, f0 >> 57); - b -= n < 1000000000; - const uint64_t f2 = (f0 & mask57) * 100; - jeaiii_write_dd(b + 2, f2 >> 57); - const uint64_t f4 = (f2 & mask57) * 100; - jeaiii_write_dd(b + 4, f4 >> 57); - const uint64_t f6 = (f4 & mask57) * 100; - jeaiii_write_dd(b + 6, f6 >> 57); - const uint64_t f8 = (f6 & mask57) * 100; - jeaiii_write_dd(b + 8, f8 >> 57); - return b + 10; -} - -simdjson_really_inline char *write_uint_jeaiii(char *b, uint64_t n) noexcept { - if (n < 100000000) { - return jeaiii_lt1e8(b, uint32_t(n)); - } - if (n < (uint64_t(1) << 32)) { - return jeaiii_9_or_10(b, n); - } - // At least 10 digits: the low 8 digits, and 2 to 12 digits above them. - const uint32_t z = uint32_t(n % 100000000); - uint64_t u = n / 100000000; - if (u < 100000000) { - // u has 2 to 8 digits (if u < 10, n would be below 2^32). - b = jeaiii_lt1e8(b, uint32_t(u)); - } else if (u < (uint64_t(1) << 32)) { - b = jeaiii_9_or_10(b, u); - } else { - // u has 11 or 12 digits: split off 8 more. - const uint32_t y = uint32_t(u % 100000000); - u /= 100000000; - b = jeaiii_lt1e8(b, uint32_t(u)); // 3 or 4 digits - b = jeaiii_8_digits(b, y); - } - return jeaiii_8_digits(b, z); -} - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_JEAIII_H -/* end file simdjson/internal/jeaiii.h */ -/* including simdjson/internal/json_escape.h: #include "simdjson/internal/json_escape.h" */ -/* begin file simdjson/internal/json_escape.h */ -#ifndef SIMDJSON_INTERNAL_JSON_ESCAPE_H -#define SIMDJSON_INTERNAL_JSON_ESCAPE_H - -// Escaping of JSON strings, shared by the builder and the DOM serializer. - -/* skipped duplicate #include "simdjson/base.h" */ -#include -#include -#include -#include -#ifdef _MSC_VER -#include -#endif - -/* - * Empirically, we have found that an inlined optimization is important for - * performance. The following macros are not ideal. We should find a better - * way to inline the code. - */ - -#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ - (defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && _M_IX86_FP == 2)) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 -#endif -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 -#endif -#endif -#if defined(__loongarch_sx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX -#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 -#endif -#endif -#if defined(__loongarch_asx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX -#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1 -#endif -#endif -#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ - defined(__riscv_vector) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV -#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 -#endif -#endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_NEON -#include -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#include -#if defined(__AVX2__) -#include -#endif -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LSX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_RVV -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#include -#ifdef bool -#undef bool -#endif -#ifdef vector -#undef vector -#endif -#endif - -namespace simdjson { -namespace internal { - -// The index of the lowest set bit of a non-zero value. The escape code runs -// outside any implementation, so it cannot use their trailing_zeroes. -simdjson_inline int trailing_zeroes(uint64_t input_num) noexcept { -#ifdef _MSC_VER - unsigned long ret; -#if defined(_M_X64) || defined(_M_ARM64) - _BitScanForward64(&ret, input_num); -#else - if (!_BitScanForward(&ret, (unsigned long)input_num)) { - _BitScanForward(&ret, (unsigned long)(input_num >> 32)); - ret += 32; - } -#endif - return int(ret); -#else - return __builtin_ctzll(input_num); -#endif -} - -// The tables are static members of a class template rather than namespace-scope -// statics: a static object would be a different object in every translation -// unit, and inline functions that index it would break the one-definition rule. -template struct escape_tables { - // 1 for the bytes that must be escaped: '"', '\\' and control characters. - static constexpr uint8_t quotable[256] = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - // The escape sequence of each control character. It is const rather than - // constexpr: MSVC rejects a constexpr char[32][7] member of a class template - // initialized from string literals (C2131). - static const char control[32][7]; -}; - -#if !SIMDJSON_CPLUSPLUS17 -template constexpr uint8_t escape_tables::quotable[256]; -#endif -template const char escape_tables::control[32][7] = { - "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", - "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r", - "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014", - "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b", - "\\u001c", "\\u001d", "\\u001e", "\\u001f"}; - -// All Unicode characters may be placed within the quotation marks, except for -// the characters that MUST be escaped: quotation mark, reverse solidus, and the -// control characters (U+0000 through U+001F). There are two-character sequence -// escape representations of some popular characters: -// \", \\, \b, \f, \n, \r, \t. -simdjson_inline void escape_json_char(char c, char *&out) { - if (c == '"') { - memcpy(out, "\\\"", 2); - out += 2; - } else if (c == '\\') { - memcpy(out, "\\\\", 2); - out += 2; - } else { - const char *v = escape_tables<>::control[uint8_t(c)]; - const size_t n = v[1] == 'u' ? 6 : 2; - memcpy(out, v, n); - out += n; - } -} - -/** - -A possible SWAR implementation of has_json_escapable_byte. It is not used -because it is slower than the current implementation. It is kept here for -reference (to show that we tried it). - -inline bool has_json_escapable_byte(uint64_t x) { - uint64_t is_ascii = 0x8080808080808080ULL & ~x; - uint64_t xor2 = x ^ 0x0202020202020202ULL; - uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; - uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; - uint64_t eq92 = (sub92 - 0x0101010101010101ULL); - return ((lt32_or_eq34 | eq92) & is_ascii) != 0; -} - -**/ - -/* - * Every variant of the escaper lives in its own namespace (escape_scalar, - * escape_sse2, escape_avx2, ...), whose content does not depend on compiler - * flags. Callers use escape_best, defined at the end of this file. - */ - -// Scalar fallback for finding next quotable character -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character_scalar(const std::string_view view, - size_t location) noexcept { - for (auto pos = view.begin() + location; pos != view.end(); ++pos) { - if (escape_tables<>::quotable[static_cast(*pos)]) { - return pos - view.begin(); - } - } - return size_t(view.size()); -} - -// Writes the escaped version of input to out, returning the number of bytes -// written. Finder::find(input, location) returns the position of the next -// quotable character at or after location, or input.size(). -template -inline size_t write_string_escaped_with(const std::string_view input, char *out) { - size_t mysize = input.size(); - - // Use SIMD position finder directly - it returns mysize if no escape needed - size_t location = Finder::find(input, 0); - if (location == mysize) { - // Fast path: no escaping needed - memcpy(out, input.data(), input.size()); - return input.size(); - } - - const char *const initout = out; - memcpy(out, input.data(), location); - out += location; - escape_json_char(input[location], out); - location += 1; - while (location < mysize) { - size_t newlocation = Finder::find(input, location); - memcpy(out, input.data() + location, newlocation - location); - out += newlocation - location; - location = newlocation; - if (location == mysize) { - break; - } - escape_json_char(input[location], out); - location += 1; - } - return size_t(out - initout); -} - -namespace escape_scalar { -struct finder { - static simdjson_inline size_t find(const std::string_view view, - size_t location) noexcept { - return find_next_json_quotable_character_scalar(view, location); - } -}; - -inline size_t write_string_escaped(const std::string_view input, char *out) { - return write_string_escaped_with(input, out); -} -} // namespace escape_scalar - -// SIMD-accelerated position finding that directly locates the first quotable -// character, combining detection and position extraction in a single pass to -// minimize redundant work. -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -namespace escape_lasx { -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"' - __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\' - __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold - - while (remaining >= 32) { - __m256i word = __lasx_xvld(ptr, 0); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __m256i needs_escape = __lasx_xvseq_b(word, v34); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92)); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32)); - - if (!__lasx_xbz_v(needs_escape)) { - // Found a quotable character - locate it via the four 64-bit lanes - uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0); - uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1); - uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2); - uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3); - size_t offset = ptr - reinterpret_cast(view.data()); - if (lane0 != 0) { - return offset + trailing_zeroes(lane0) / 8; - } else if (lane1 != 0) { - return offset + 8 + trailing_zeroes(lane1) / 8; - } else if (lane2 != 0) { - return offset + 16 + trailing_zeroes(lane2) / 8; - } else { - return offset + 24 + trailing_zeroes(lane3) / 8; - } - } - ptr += 32; - remaining -= 32; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -struct finder { - static simdjson_inline size_t find(const std::string_view view, - size_t location) noexcept { - return find_next_json_quotable_character(view, location); - } -}; - -inline size_t write_string_escaped(const std::string_view input, char *out) { - return write_string_escaped_with(input, out); -} -} // namespace escape_lasx -#endif - -#if SIMDJSON_EXPERIMENTAL_HAS_LSX -namespace escape_lsx { -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - //SIMD constants for characters requiring escape - __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' - __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' - __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold - - while (remaining >= 16){ - __m128i word = __lsx_vld(ptr, 0); - - //Check for the quotable characters: '"', '\\', or control char (<32) - __m128i needs_escape = __lsx_vseq_b(word, v34); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); - - if (!__lsx_bz_v(needs_escape)){ - - //Found quotable character - extract exact byte position - uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); - uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); - size_t offset = ptr - reinterpret_cast(view.data()); - if ( lo != 0) { - return offset + trailing_zeroes(lo) / 8; - } else { - return offset + 8 + trailing_zeroes(hi) / 8; - } - } - ptr += 16; - remaining -= 16; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -struct finder { - static simdjson_inline size_t find(const std::string_view view, - size_t location) noexcept { - return find_next_json_quotable_character(view, location); - } -}; - -inline size_t write_string_escaped(const std::string_view input, char *out) { - return write_string_escaped_with(input, out); -} -} // namespace escape_lsx -#endif - -#if SIMDJSON_EXPERIMENTAL_HAS_RVV -namespace escape_rvv { -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - while (remaining > 0) { - size_t vl = __riscv_vsetvl_e8m1(remaining); - vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmseq(word, (uint8_t)92, vl), vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmsltu(word, (uint8_t)32, vl), vl); - - long first = __riscv_vfirst(needs_escape, vl); - if (first >= 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + first; - } - ptr += vl; - remaining -= vl; - } - - return len; -} -struct finder { - static simdjson_inline size_t find(const std::string_view view, - size_t location) noexcept { - return find_next_json_quotable_character(view, location); - } -}; - -inline size_t write_string_escaped(const std::string_view input, char *out) { - return write_string_escaped_with(input, out); -} -} // namespace escape_rvv -#endif - -#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 -namespace escape_ppc64 { -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' - __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' - __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold - - // Bitmask for vec_vbpermq to extract one bit per byte - const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, - 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, - 0x18, 0x10, 0x08, 0x00}; - - while (remaining >= 16) { - __vector unsigned char word = - vec_vsx_ld(0, reinterpret_cast(ptr)); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __vector unsigned char needs_escape = - (__vector unsigned char)vec_cmpeq(word, v34); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmpeq(word, v92)); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmplt(word, v32)); - - __vector unsigned long long result = - (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); -#ifdef __LITTLE_ENDIAN__ - unsigned int mask = static_cast(result[1]); -#else - unsigned int mask = static_cast(result[0]); -#endif - if (mask != 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + __builtin_ctz(mask); - } - ptr += 16; - remaining -= 16; - } - - // Scalar fallback for remaining bytes - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -struct finder { - static simdjson_inline size_t find(const std::string_view view, - size_t location) noexcept { - return find_next_json_quotable_character(view, location); - } -}; - -inline size_t write_string_escaped(const std::string_view input, char *out) { - return write_string_escaped_with(input, out); -} -} // namespace escape_ppc64 -#endif - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON -// Block escaping: write_string_escaped scans the input itself, 16 bytes at a -// time. The two instruction sets are never available together. -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 -namespace escape_sse2 { -#else -namespace escape_neon { -#endif - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 - -using escape_vector = __m128i; -// Mask bits that each input byte contributes to escape_bitmask(). -static constexpr unsigned escape_mask_bits = 1; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return _mm_loadu_si128(reinterpret_cast(p)); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v); -} - -// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b. -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - return _mm_unpacklo_epi64( - _mm_loadl_epi64(reinterpret_cast(a)), - _mm_loadl_epi64(reinterpret_cast(b))); -} - -// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The -// upper half is unspecified; callers only look at the low eight bits. -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - int32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32)); -} - -// Sets every bit of byte k when byte k of v is a quotable character ('"', -// '\\' or a control character). -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - const __m128i v34 = _mm_set1_epi8(34); // '"' - const __m128i v92 = _mm_set1_epi8(92); // '\\' - const __m128i v31 = _mm_set1_epi8(31); // for control char detection - __m128i needs_escape = _mm_cmpeq_epi8(v, v34); - needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92)); - return _mm_or_si128( - needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128())); -} - -// True when any byte needs escaping. Kept separate from escape_bitmask -// because some instruction sets can answer it without leaving the vector -// register file; on SSE2 the compiler folds the two together. -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return _mm_movemask_epi8(flags) != 0; -} - -// A 16-bit mask with bit k set when byte k needed escaping. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - return uint64_t(uint32_t(_mm_movemask_epi8(flags))); -} - -#else // SIMDJSON_EXPERIMENTAL_HAS_NEON - -using escape_vector = uint8x16_t; -static constexpr unsigned escape_mask_bits = 4; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return vld1q_u8(p); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - vst1q_u8(reinterpret_cast(out), v); -} - -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint64_t a64, b64; - memcpy(&a64, a, 8); - memcpy(&b64, b, 8); - return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1)); -} - -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1)); -} - -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"' - needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\' - return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32))); -} - -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0; -} - -// Four bits per byte rather than one: escape_block and the tail paths scale -// their shifts by escape_mask_bits to match. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4); - return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0); -} - -#endif // instruction set selection -// The escape bitmask of a 16-byte block. -simdjson_inline uint64_t escape_mask(escape_vector v) noexcept { - return escape_bitmask(escape_flags(v)); -} - -// Copies n bytes with n < 16, using overlapping loads and stores. It never -// reads more than n bytes from src, nor writes more than n bytes to dst. -simdjson_inline void copy_lt16(char *dst, const uint8_t *src, - size_t n) noexcept { - if (n >= 8) { - memcpy(dst, src, 8); - memcpy(dst + n - 8, src + n - 8, 8); - } else if (n >= 4) { - memcpy(dst, src, 4); - memcpy(dst + n - 4, src + n - 4, 4); - } else if (n > 0) { - dst[0] = char(src[0]); - dst[n >> 1] = char(src[n >> 1]); - dst[n - 1] = char(src[n - 1]); - } -} - -// Escapes the bytes of src in the range [i, blockend), given that m is the -// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m -// at byte k is non-zero when src[i + k] requires escaping. Returns the updated -// output pointer. -simdjson_never_inline char *escape_block(const uint8_t *src, char *out, - size_t i, size_t blockend, - uint64_t m) noexcept { - constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1; - - size_t pos = i; // first byte not yet copied - while (m) { - const size_t tz = trailing_zeroes(m); - const size_t next = i + tz / escape_mask_bits; - // Copy the run of safe bytes that precedes this escape. - copy_lt16(out, src + pos, next - pos); - out += next - pos; - escape_json_char(char(src[next]), out); - pos = next + 1; - m &= ~(lane << tz); - } - // Copy whatever follows the last escape. - copy_lt16(out, src + pos, blockend - pos); - return out + (blockend - pos); -} - -// Escapes src[i, len) to out, returning the updated output pointer. The input -// must be the whole string: when len >= 16, the tail reloads its last 16 bytes. -simdjson_really_inline char *escape_from(const uint8_t *src, size_t len, - size_t i, char *out) noexcept { - while (i + 16 <= len) { - escape_vector word = escape_load16(src + i); - escape_vector flags = escape_flags(word); - if (simdjson_likely(!escape_any(flags))) { - escape_store16(out, word); - out += 16; - } else { - out = escape_block(src, out, i, i + 16, escape_bitmask(flags)); - } - i += 16; - } - if (i < len) { - const size_t rem = len - i; - uint64_t m; - if (len >= 16) { - // The last 16 bytes of the input are in bounds. Bit k of that block's - // mask belongs to input position len - 16 + k, so shift it down to align - // bit 0 with position i. - m = escape_mask(escape_load16(src + len - 16)) >> - (escape_mask_bits * (16 - rem)); - } else if (len >= 8) { - // Here i == 0 and rem == len. Two overlapping 8-byte loads cover - // [0, 8) and [len - 8, len), which is the whole input since len < 16. - uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8)); - constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1; - m = (mm & low8) | - ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8))); - } else if (len >= 4) { - // Same idea with two overlapping 4-byte loads. - uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4)); - constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1; - m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4) - << (escape_mask_bits * (len - 4))); - } else { - // Fewer than 4 bytes: at most three table lookups, no need for SIMD. - for (size_t k = 0; k < len; k++) { - uint8_t c = src[k]; - if (escape_tables<>::quotable[c]) { - escape_json_char(char(c), out); - } else { - *out++ = char(c); - } - } - return out; - } - if (m == 0) { - copy_lt16(out, src + i, rem); - out += rem; - } else { - out = escape_block(src, out, i, len, m); - } - } - return out; -} - -// Writes the escaped version of input to out, returning the number of bytes -// written. -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) { - const uint8_t *src = reinterpret_cast(input.data()); - return size_t(escape_from(src, input.size(), 0, out) - out); -} - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 -} // namespace escape_sse2 -#else -} // namespace escape_neon -#endif -#endif // SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__) -// SSE2 with a 32-byte main loop. It has its own namespace because it is -// compiled only when __AVX2__ is defined. -namespace escape_avx2 { -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) { - using namespace escape_sse2; - const size_t len = input.size(); - const uint8_t *src = reinterpret_cast(input.data()); - const char *const initout = out; - - size_t i = 0; - while (i + 32 <= len) { - const __m256i word = _mm256_loadu_si256(reinterpret_cast(src + i)); - const __m256i flags = _mm256_or_si256( - _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"' - _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\' - _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)), - _mm256_setzero_si256())); // control - const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags)); - if (simdjson_likely(mask == 0)) { - _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word); - out += 32; - } else { - for (size_t half = 0; half < 32; half += 16) { - const uint64_t m = (mask >> half) & 0xFFFF; - if (m == 0) { - escape_store16(out, escape_load16(src + i + half)); - out += 16; - } else { - out = escape_block(src, out, i + half, i + half + 16, m); - } - } - } - i += 32; - } - return size_t(escape_from(src, len, i, out) - initout); -} -} // namespace escape_avx2 -#endif - -// The variant that the builder and the DOM serializer use: the best one that -// the compiler flags allow. Translation units compiled with different flags -// may select different variants for the same inline functions; see -// "Mixing compiler flags" in doc/implementation-selection.md. -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__) -namespace escape_best = escape_avx2; -#elif SIMDJSON_EXPERIMENTAL_HAS_SSE2 -namespace escape_best = escape_sse2; -#elif SIMDJSON_EXPERIMENTAL_HAS_NEON -namespace escape_best = escape_neon; -#elif SIMDJSON_EXPERIMENTAL_HAS_LASX -namespace escape_best = escape_lasx; -#elif SIMDJSON_EXPERIMENTAL_HAS_LSX -namespace escape_best = escape_lsx; -#elif SIMDJSON_EXPERIMENTAL_HAS_RVV -namespace escape_best = escape_rvv; -#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 -namespace escape_best = escape_ppc64; -#else -namespace escape_best = escape_scalar; -#endif - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_JSON_ESCAPE_H -/* end file simdjson/internal/json_escape.h */ -/* skipped duplicate #include "simdjson/internal/tape_type.h" */ - -/* skipped duplicate #include "simdjson/dom/array-inl.h" */ -/* skipped duplicate #include "simdjson/dom/object-inl.h" */ -/* skipped duplicate #include "simdjson/internal/tape_ref-inl.h" */ - -#include -#include - -namespace simdjson { -namespace dom { -inline bool parser::print_json(std::ostream &os) const noexcept { - if (!valid) { - return false; - } - simdjson::internal::string_builder<> sb; - sb.append(doc.root()); - std::string_view answer = sb.str(); - os << answer; - return true; -} - -inline std::ostream &operator<<(std::ostream &out, - simdjson::dom::element value) { - simdjson::internal::string_builder<> sb; - sb.append(value); - return (out << sb.str()); -} -#if SIMDJSON_EXCEPTIONS -inline std::ostream & -operator<<(std::ostream &out, - simdjson::simdjson_result x) { - if (x.error()) { - throw simdjson::simdjson_error(x.error()); - } - return (out << x.value()); -} -#endif -inline std::ostream &operator<<(std::ostream &out, simdjson::dom::array value) { - simdjson::internal::string_builder<> sb; - sb.append(value); - return (out << sb.str()); -} -#if SIMDJSON_EXCEPTIONS -inline std::ostream & -operator<<(std::ostream &out, - simdjson::simdjson_result x) { - if (x.error()) { - throw simdjson::simdjson_error(x.error()); - } - return (out << x.value()); -} -#endif -inline std::ostream &operator<<(std::ostream &out, - simdjson::dom::object value) { - simdjson::internal::string_builder<> sb; - sb.append(value); - return (out << sb.str()); -} -#if SIMDJSON_EXCEPTIONS -inline std::ostream & -operator<<(std::ostream &out, - simdjson::simdjson_result x) { - if (x.error()) { - throw simdjson::simdjson_error(x.error()); - } - return (out << x.value()); -} -#endif - -} // namespace dom - -namespace internal { - -/*** - * Minifier/formatter code. - **/ - -template -simdjson_inline void base_formatter::number(uint64_t x) { - // write_uint_jeaiii may store one byte before its output pointer. - char number_buffer[24]; - chars(number_buffer + 1, write_uint_jeaiii(number_buffer + 1, x)); -} - -template -simdjson_inline void base_formatter::number(int64_t x) { - char number_buffer[24]; - // 0 - uint64_t(x) is the magnitude, also for INT64_MIN. - char *end = write_uint_jeaiii(number_buffer + 1, x < 0 ? 0 - uint64_t(x) : uint64_t(x)); - number_buffer[0] = '-'; - chars(number_buffer + (x >= 0), end); -} - -template -simdjson_inline void base_formatter::number(double x) { -#if SIMDJSON_ENABLE_NAN_INF - if (simdjson_unlikely(!std::isfinite(x))) { - if (std::isnan(x)) { - char const *s = "NaN"; - chars(s, s + 3); - } else { - if (x < 0) { - one_char('-'); - } - char const *s = "Infinity"; - chars(s, s + 8); - } - return; - } -#endif - - // Must be to_chars_buffer_size (40): only ~24 chars are emitted, but - // to_chars over-writes with fixed-size 16/17-byte copies for inlining. - char number_buffer[simdjson::internal::to_chars_buffer_size]; - // Currently, passing the nullptr to the second argument is - // safe because our implementation does not check the second - // argument. - char *newp = simdjson::internal::to_chars(number_buffer, nullptr, x); - chars(number_buffer, newp); -} - -template -simdjson_inline void base_formatter::start_array() { - one_char('['); -} - -template -simdjson_inline void base_formatter::end_array() { - one_char(']'); -} - -template -simdjson_inline void base_formatter::start_object() { - one_char('{'); -} - -template -simdjson_inline void base_formatter::end_object() { - one_char('}'); -} - -template -simdjson_inline void base_formatter::comma() { - one_char(','); -} - -template -simdjson_inline void base_formatter::true_atom() { - const char *s = "true"; - chars(s, s + 4); -} - -template -simdjson_inline void base_formatter::false_atom() { - const char *s = "false"; - chars(s, s + 5); -} - -template -simdjson_inline void base_formatter::null_atom() { - const char *s = "null"; - chars(s, s + 4); -} - -template -simdjson_inline void base_formatter::one_char(char c) { - buffer.push_back(c); -} - -template -simdjson_inline void base_formatter::chars(const char *begin, - const char *end) { - buffer.append(begin, end); -} - -template -simdjson_inline void -base_formatter::key(std::string_view unescaped) { - string(unescaped); - one_char(':'); -} - -template -simdjson_inline void -base_formatter::string(std::string_view unescaped) { - one_char('\"'); - // Each input byte becomes at most 6 output bytes. Escaping the string in - // pieces bounds the room reserved ahead of a long string. - constexpr size_t piece = 4096; - for (size_t i = 0; i < unescaped.size(); i += piece) { - const std::string_view part = unescaped.substr(i, piece); - char *out = buffer.reserve(6 * part.size()); - buffer.set_end(out + escape_best::write_string_escaped(part, out)); - } - one_char('\"'); -} - -template inline void base_formatter::clear() { - buffer.clear(); -} - -template -simdjson_inline std::string_view base_formatter::str() const { - return buffer.str(); -} - -simdjson_inline void mini_formatter::print_newline() { return; } - -simdjson_inline void mini_formatter::print_indents(size_t depth) { - (void)depth; - return; -} - -simdjson_inline void mini_formatter::print_space() { return; } - -simdjson_inline void pretty_formatter::print_newline() { one_char('\n'); } - -simdjson_inline void pretty_formatter::print_indents(size_t depth) { - if (indent_step <= 0) { - return; - } - buffer.append_n(' ', depth * static_cast(indent_step)); -} - -simdjson_inline void pretty_formatter::print_space() { one_char(' '); } - -/*** - * String building code. - **/ - -template -inline void string_builder::append(simdjson::dom::element value) { - append(value, 0); - format.print_newline(); -} - -template -inline void string_builder::append(simdjson::dom::element value, size_t base_depth) { - // using tape_type = simdjson::internal::tape_type; - size_t depth = 0; - constexpr size_t MAX_DEPTH = 16; - bool is_object[MAX_DEPTH]; - is_object[0] = false; - bool after_value = false; - - internal::tape_ref iter(value.tape); - do { - // print commas after each value - if (after_value) { - format.comma(); - format.print_newline(); - } - - // At depth 0 the caller has already indented the value. - if (depth != 0) { - format.print_indents(base_depth + depth); - } - - // If we are in an object, print the next key and :, and skip to the next - // value. - if (is_object[depth]) { - format.key(iter.get_string_view()); - format.print_space(); - iter.json_index++; - } - switch (iter.tape_ref_type()) { - - // Arrays - case tape_type::START_ARRAY: { - // If we're too deep, we need to recurse to go deeper. - depth++; - if (simdjson_unlikely(depth >= MAX_DEPTH)) { - append(simdjson::dom::element(iter), base_depth + depth - 1); - iter.json_index = iter.matching_brace_index() - 1; // Jump to the ] - depth--; - break; - } - - // Output start [ - format.start_array(); - iter.json_index++; - - // Handle empty [] (we don't want to come back around and print commas) - if (iter.tape_ref_type() == tape_type::END_ARRAY) { - format.end_array(); - depth--; - break; - } - - is_object[depth] = false; - after_value = false; - format.print_newline(); - continue; - } - - // Objects - case tape_type::START_OBJECT: { - // If we're too deep, we need to recurse to go deeper. - depth++; - if (simdjson_unlikely(depth >= MAX_DEPTH)) { - append(simdjson::dom::element(iter), base_depth + depth - 1); - iter.json_index = iter.matching_brace_index() - 1; // Jump to the } - depth--; - break; - } - - // Output start { - format.start_object(); - iter.json_index++; - - // Handle empty {} (we don't want to come back around and print commas) - if (iter.tape_ref_type() == tape_type::END_OBJECT) { - format.end_object(); - depth--; - break; - } - - is_object[depth] = true; - after_value = false; - format.print_newline(); - continue; - } - - // Scalars - case tape_type::STRING: - format.string(iter.get_string_view()); - break; - case tape_type::BIGINT: { - // Big integer stored as string -- output raw digits (no quotes) - auto sv = iter.get_string_view(); - format.chars(sv.data(), sv.data() + sv.size()); - break; - } - case tape_type::INT64: - format.number(iter.next_tape_value()); - iter.json_index++; // numbers take up 2 spots, so we need to increment - // extra - break; - case tape_type::UINT64: - format.number(iter.next_tape_value()); - iter.json_index++; // numbers take up 2 spots, so we need to increment - // extra - break; - case tape_type::DOUBLE: - format.number(iter.next_tape_value()); - iter.json_index++; // numbers take up 2 spots, so we need to increment - // extra - break; - case tape_type::TRUE_VALUE: - format.true_atom(); - break; - case tape_type::FALSE_VALUE: - format.false_atom(); - break; - case tape_type::NULL_VALUE: - format.null_atom(); - break; - - // These are impossible - case tape_type::END_ARRAY: - case tape_type::END_OBJECT: - case tape_type::ROOT: - SIMDJSON_UNREACHABLE(); - } - iter.json_index++; - after_value = true; - - // Handle multiple ends in a row - while (depth != 0 && (iter.tape_ref_type() == tape_type::END_ARRAY || - iter.tape_ref_type() == tape_type::END_OBJECT)) { - format.print_newline(); - depth--; - format.print_indents(base_depth + depth); - if (iter.tape_ref_type() == tape_type::END_ARRAY) { - format.end_array(); - } else { - format.end_object(); - } - iter.json_index++; - } - - // Stop when we're at depth 0 - } while (depth != 0); -} - -template -inline void string_builder::append(simdjson::dom::object value) { - append(simdjson::dom::element(value)); -} - -template -inline void string_builder::append(simdjson::dom::array value) { - append(simdjson::dom::element(value)); -} - -template -simdjson_inline void -string_builder::append(simdjson::dom::key_value_pair kv) { - format.key(kv.key); - append(kv.value); -} - -template -simdjson_inline void string_builder::clear() { - format.clear(); -} - -template -simdjson_inline std::string_view string_builder::str() const { - return format.str(); -} - -} // namespace internal -} // namespace simdjson - -#endif -/* end file simdjson/dom/serialization-inl.h */ -/* including simdjson/dom/fractured_json-inl.h: #include "simdjson/dom/fractured_json-inl.h" */ -/* begin file simdjson/dom/fractured_json-inl.h */ -#ifndef SIMDJSON_DOM_FRACTURED_JSON_INL_H -#define SIMDJSON_DOM_FRACTURED_JSON_INL_H - -/* skipped duplicate #include "simdjson/dom/fractured_json.h" */ -/* skipped duplicate #include "simdjson/dom/serialization.h" */ -/* skipped duplicate #include "simdjson/dom/element-inl.h" */ -/* skipped duplicate #include "simdjson/dom/array-inl.h" */ -/* skipped duplicate #include "simdjson/dom/object-inl.h" */ -/* skipped duplicate #include "simdjson/dom/parser-inl.h" */ -/* skipped duplicate #include "simdjson/padded_string.h" */ -/* including simdjson/internal/json_structure_analyzer.h: #include "simdjson/internal/json_structure_analyzer.h" */ -/* begin file simdjson/internal/json_structure_analyzer.h */ -#ifndef SIMDJSON_INTERNAL_JSON_STRUCTURE_ANALYZER_H -#define SIMDJSON_INTERNAL_JSON_STRUCTURE_ANALYZER_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/element.h" */ -/* skipped duplicate #include "simdjson/dom/array.h" */ -/* skipped duplicate #include "simdjson/dom/object.h" */ -/* skipped duplicate #include "simdjson/dom/fractured_json.h" */ -/* skipped duplicate #include "simdjson/internal/tape_type.h" */ - -#include -#include -#include -#include - -namespace simdjson { -namespace internal { - -/** - * Layout mode for fractured JSON formatting. - */ -enum class layout_mode { - single_line, // Single line: [1, 2, 3] or {"a": 1} - compact_multiline, // Multiple items per line with breaks - table, // Tabular format for arrays of similar objects - expanded // Traditional multi-line with indentation -}; - -/** Kind of value found in a table column across all rows that have one. */ -enum class table_column_type { - unknown, - simple, // string, bool, or null - number, - array, - object, - mixed // rows disagree on kind -}; - -/** Column of a table-formatted array.*/ -struct table_column { - /** Column name for object rows (may be the empty string). */ - std::string key{}; - /** True for object rows (the column has a key), false for array rows. */ - bool has_key = false; - /** Rendered length of key */ - size_t key_width = 0; - table_column_type type = table_column_type::unknown; - /** Widest rendered value in this column (when fully expanding all children) */ - size_t width = 0; - /** Widest plain value in this column. */ - size_t plain_width = 0; - /** subcolumns (only populated if every child is an array or every child is an object) */ - std::vector children{}; -}; - -/** Whether rows from this column list use nested vs. simple bracket padding - * One shared decision, since picking it per-row would misalign width-aligned rows. */ -inline bool table_row_is_nested(const std::vector& columns) { - for (const table_column& col : columns) { - if (col.type == table_column_type::object || col.type == table_column_type::array) { - return true; - } - } - return false; -} - -/** - * Metrics computed for a JSON element during structure analysis. - * These metrics drive layout decisions and contain child metrics for recursive formatting. - */ -struct element_metrics { - /** Nesting depth score (0 = scalar, 1 = flat container, etc.) */ - size_t complexity = 0; - - /** Estimated character length if rendered inline (minified + spaces) */ - size_t estimated_inline_len = 0; - - /** Number of direct children (0 for scalars) */ - size_t child_count = 0; - - /** Pre-computed: can this element be rendered inline? */ - bool can_inline = false; - - /** Is this an array where all elements have similar structure? */ - bool is_uniform_array = false; - - /** for uniform arrays: this array's columns for alignment */ - std::vector table_columns{}; - /** Widest table row after pruning recursive columns that don't fit into line budget */ - size_t table_row_width = 0; - /** Widest table row without pruning recursive columns that don't fit into line budget */ - size_t table_row_width_full = 0; - - /** Child metrics for arrays and objects (in order of iteration) */ - std::vector children{}; - - /** For scalar uniform arrays (table_columns empty): the rows' common type. */ - table_column_type scalar_column_type = table_column_type::unknown; -}; - -/** children[idx], or a default-constructed element_metrics if idx is out of range */ -inline const element_metrics& child_metrics_at(const std::vector& children, size_t idx) { - static const element_metrics empty{}; - return idx < children.size() ? children[idx] : empty; -} - -/** *ptr, or a default-constructed element_metrics if ptr is null. */ -inline const element_metrics& child_metrics_at(const element_metrics* ptr) { - static const element_metrics empty{}; - return ptr ? *ptr : empty; -} - -/** - * Analyzes JSON structure to compute metrics for formatting decisions. - * - * The analyzer performs a single pass over the DOM to compute: - * - Complexity (nesting depth) - * - Estimated inline length - * - Array uniformity for table detection - * - * Metrics are stored hierarchically with child metrics embedded in parent metrics, - * enabling efficient lookup during formatting without address-based caching. - */ -class structure_analyzer { -public: - /** Default constructor */ - structure_analyzer() : current_opts_(nullptr) {} - - /** Copy constructor - deleted since class has pointer member */ - structure_analyzer(const structure_analyzer&) = delete; - - /** Copy assignment - deleted since class has pointer member */ - structure_analyzer& operator=(const structure_analyzer&) = delete; - - /** Move constructor */ - structure_analyzer(structure_analyzer&&) = default; - - /** Move assignment */ - structure_analyzer& operator=(structure_analyzer&&) = default; - - /** - * Analyze a DOM element and compute metrics. - * @param elem The element to analyze - * @param opts Formatting options that affect metric computation - * @return Metrics for the root element (with child metrics embedded) - */ - element_metrics analyze(const dom::element& elem, - const fractured_json_options& opts); - - /** - * Clear state. - */ - void clear(); - - /** - * Analyze an array element directly (for standalone array formatting). - * @param arr The array to analyze - * @param opts Formatting options - * @return Metrics for the array - */ - element_metrics analyze_array(const dom::array& arr, - const fractured_json_options& opts); - - /** - * Analyze an object element directly (for standalone object formatting). - * @param obj The object to analyze - * @param opts Formatting options - * @return Metrics for the object - */ - element_metrics analyze_object(const dom::object& obj, - const fractured_json_options& opts); - - /** Decide layout at the given render depth. Kept out of analysis since - * the same metrics can render inline or expanded at different depths. - * key_width is the width of the key, colon and colon padding written - * before the value on its first line (0 if the value has no key). */ - static layout_mode decide_layout(const element_metrics& metrics, - size_t depth, - const fractured_json_options& opts, - bool has_trailing_comma = false, - size_t key_width = 0); - - /** Estimate inline length for a string (including quotes and escaping) */ - size_t estimate_string_length(std::string_view s) const; - -private: - const fractured_json_options* current_opts_ = nullptr; - - /** Recursive analysis implementation */ - element_metrics analyze_element(const dom::element& elem, size_t depth) const; - - /** Analyze scalar values (strings, numbers, booleans, null) */ - element_metrics analyze_scalar(const dom::element& elem) const; - - /** Analyze an array element */ - element_metrics analyze_array(const dom::array& arr, size_t depth) const; - - /** Analyze an object element */ - element_metrics analyze_object(const dom::object& obj, size_t depth) const; - - /** Estimate inline length for a number */ - size_t estimate_number_length(double d) const; - size_t estimate_number_length(int64_t i) const; - size_t estimate_number_length(uint64_t u) const; - - /** - * Check if an array contains uniformly-shaped rows suitable for table - * formatting, filling in corresponding metrics - * @param arr The array to check - * @param metrics The array's metrics, with children already filled in - * @param depth The array's depth - */ - bool check_array_uniformity(const dom::array& arr, element_metrics& metrics, size_t depth) const; - - static table_column_type classify_table_value(dom::element_type type); - - /** Find common type across a set of sibling values and the widest of their rendered lengths */ - static void classify_and_measure(const std::vector>& values, - table_column_type& common, size_t& max_width); - - /** Recursively build table columns for an array */ - void build_table_columns(const std::vector>& values, - std::vector& out_columns) const; - - /** Rendered width of a row, assuming each column's current */ - size_t compute_columns_width(const std::vector& columns) const; - - /** Height of a column's recursion (0 = leaf). */ - static size_t column_height(const table_column& column); - - /** Flatten deepest columns of the table */ - static bool flatten_deepest_columns(std::vector& columns); - - /** Bottom-up refresh of column widths after flatten_deepest_columns. */ - void recompute_column_widths(std::vector& columns) const; -}; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_JSON_STRUCTURE_ANALYZER_H -/* end file simdjson/internal/json_structure_analyzer.h */ -/* including simdjson/internal/fractured_formatter.h: #include "simdjson/internal/fractured_formatter.h" */ -/* begin file simdjson/internal/fractured_formatter.h */ -#ifndef SIMDJSON_INTERNAL_FRACTURED_FORMATTER_H -#define SIMDJSON_INTERNAL_FRACTURED_FORMATTER_H - -/* skipped duplicate #include "simdjson/dom/serialization.h" */ -/* skipped duplicate #include "simdjson/dom/fractured_json.h" */ -/* skipped duplicate #include "simdjson/internal/json_structure_analyzer.h" */ - -namespace simdjson { -namespace internal { - -/** - * Fractured JSON formatter using CRTP pattern. - * - * This formatter intelligently chooses between different layout modes - * (inline, compact multiline, table, expanded) based on pre-computed - * structure metrics. - */ -class fractured_formatter : public base_formatter { -public: - explicit fractured_formatter(const fractured_json_options& opts = {}); - - /** CRTP hook: print newline (context-aware) */ - simdjson_inline void print_newline(); - - /** CRTP hook: print indentation */ - simdjson_inline void print_indents(size_t depth); - - /** CRTP hook: print space (context-aware) */ - simdjson_inline void print_space(); - - /** Set the current layout mode */ - void set_layout_mode(layout_mode mode); - - /** Get the current layout mode */ - layout_mode get_layout_mode() const; - - /** Track current line length for compact multiline decisions */ - void track_line_length(size_t chars); - - /** Check if we should break to a new line in compact mode */ - bool should_break_line(size_t upcoming_length) const; - - /** Get the options */ - const fractured_json_options& options() const; - -private: - fractured_json_options options_; - layout_mode current_layout_ = layout_mode::expanded; - size_t current_line_length_ = 0; -}; - -/** RAII helper forcing single line layout and restoring previous mode on exit */ -class scoped_single_line_mode { -public: - explicit scoped_single_line_mode(fractured_formatter& format) - : format_(format), prev_(format.get_layout_mode()) { - format_.set_layout_mode(layout_mode::single_line); - } - ~scoped_single_line_mode() { format_.set_layout_mode(prev_); } - scoped_single_line_mode(const scoped_single_line_mode&) = delete; - scoped_single_line_mode& operator=(const scoped_single_line_mode&) = delete; - -private: - fractured_formatter& format_; - layout_mode prev_; -}; - -/** - * Specialized string builder for fractured JSON formatting. - * - * This builder performs two passes: - * 1. Analyze the structure to compute metrics - * 2. Format using the metrics to make layout decisions - */ -class fractured_string_builder { -public: - fractured_string_builder(const fractured_json_options& opts = {}); - - /** Append a DOM element with fractured formatting */ - void append(const dom::element& value); - - /** Append a DOM array with fractured formatting */ - void append(const dom::array& value); - - /** Append a DOM object with fractured formatting */ - void append(const dom::object& value); - - /** Clear the builder */ - simdjson_inline void clear(); - - /** Get the formatted string */ - simdjson_inline std::string_view str() const; - -private: - fractured_formatter format_; - structure_analyzer analyzer_; - fractured_json_options options_; - - /** Format an element using pre-computed metrics */ - void format_element(const dom::element& elem, const element_metrics& metrics, size_t depth, - bool has_trailing_comma = false, size_t key_width = 0); - - /** Format an array with the appropriate layout */ - void format_array(const dom::array& arr, const element_metrics& metrics, size_t depth, - bool has_trailing_comma = false, size_t key_width = 0); - - /** Format an element on a single line, whatever its depth */ - void format_element_inline(const dom::element& elem, const element_metrics& metrics); - - /** Format an array inline: [1, 2, 3] */ - void format_array_inline(const dom::array& arr, const element_metrics& metrics); - - /** Format an array with compact multiline: multiple items per line */ - void format_array_compact_multiline(const dom::array& arr, const element_metrics& metrics, size_t depth); - - /** Like format_array_compact_multiline, but rows are cross-row aligned - * and packed using a fixed per-row slot width. */ - void format_array_compact_multiline_aligned(const dom::array& arr, const element_metrics& metrics, size_t depth); - - /** Format an array as a table */ - void format_array_as_table(const dom::array& arr, const element_metrics& metrics, size_t depth); - - /** Write one object row's columns */ - void format_table_object_row(const dom::object& obj, const element_metrics& row_metrics, - const std::vector& columns, size_t depth); - - /** Write one array row's columns */ - void format_table_array_row(const dom::array& arr, const element_metrics& row_metrics, - const std::vector& columns, size_t depth); - - /** Dispatches to format_table_object_row/format_table_array_row based on elem's type. */ - void format_table_row(const dom::element& elem, const element_metrics& row_metrics, - const std::vector& columns, size_t depth); - - /** Row for a uniform scalar array: writes elem inline, then pads to width so every row lines up. */ - void format_table_scalar_row(const dom::element& elem, const element_metrics& row_metrics, - size_t width, table_column_type column_type); - - /** Shared per-column writer: recurses if the column has children, - * otherwise writes a plain padded value or blank. */ - void format_table_row_columns(const std::vector& columns, - const std::vector& found, - const std::vector& values, - const std::vector& value_metrics, - size_t depth); - - /** Writes a single aligned leaf value */ - void format_table_leaf_value(const dom::element& elem, const element_metrics& vm, size_t width, - table_column_type column_type, bool needs_comma, - bool add_comma_space); - - /** Whether, for a column of the given type, the comma goes right after the value */ - bool comma_goes_before_padding(table_column_type column_type) const; - - /** Format an array expanded: one item per line */ - void format_array_expanded(const dom::array& arr, const element_metrics& metrics, size_t depth); - - /** Format an object with the appropriate layout */ - void format_object(const dom::object& obj, const element_metrics& metrics, size_t depth, - bool has_trailing_comma = false, size_t key_width = 0); - - /** Format an object inline: {"a": 1, "b": 2} */ - void format_object_inline(const dom::object& obj, const element_metrics& metrics); - - /** Format an object expanded: one key per line */ - void format_object_expanded(const dom::object& obj, const element_metrics& metrics, size_t depth); - - /** Format a scalar value */ - void format_scalar(const dom::element& elem); - - /** Whether to pad this container's own brackets. */ - bool bracket_padding_for(const element_metrics& metrics) const; -}; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_FRACTURED_FORMATTER_H -/* end file simdjson/internal/fractured_formatter.h */ - -#include -#include -#include -#include -#include - -namespace simdjson { -namespace internal { - -// -// Structure Analyzer Implementation -// - -inline element_metrics structure_analyzer::analyze(const dom::element& elem, - const fractured_json_options& opts) { - current_opts_ = &opts; - return analyze_element(elem, 0); -} - -inline void structure_analyzer::clear() { - current_opts_ = nullptr; -} - -inline element_metrics structure_analyzer::analyze_array(const dom::array& arr, - const fractured_json_options& opts) { - current_opts_ = &opts; - return analyze_array(arr, 0); -} - -inline element_metrics structure_analyzer::analyze_object(const dom::object& obj, - const fractured_json_options& opts) { - current_opts_ = &opts; - return analyze_object(obj, 0); -} - -inline element_metrics structure_analyzer::analyze_element(const dom::element& elem, size_t depth) const { - switch (elem.type()) { - case dom::element_type::ARRAY: { - dom::array arr; - if (elem.get_array().get(arr) == SUCCESS) { - return analyze_array(arr, depth); - } - break; - } - case dom::element_type::OBJECT: { - dom::object obj; - if (elem.get_object().get(obj) == SUCCESS) { - return analyze_object(obj, depth); - } - break; - } - default: - // Handle all scalar types with a helper - return analyze_scalar(elem); - } - return element_metrics{}; -} - -inline element_metrics structure_analyzer::analyze_scalar(const dom::element& elem) const { - element_metrics metrics; - metrics.complexity = 0; - metrics.child_count = 0; - metrics.can_inline = true; - - switch (elem.type()) { - case dom::element_type::STRING: { - std::string_view str; - if (elem.get_string().get(str) == SUCCESS) { - metrics.estimated_inline_len = estimate_string_length(str); - } - break; - } - case dom::element_type::INT64: { - int64_t val; - if (elem.get_int64().get(val) == SUCCESS) { - metrics.estimated_inline_len = estimate_number_length(val); - } - break; - } - case dom::element_type::UINT64: { - uint64_t val; - if (elem.get_uint64().get(val) == SUCCESS) { - metrics.estimated_inline_len = estimate_number_length(val); - } - break; - } - case dom::element_type::DOUBLE: { - double val; - if (elem.get_double().get(val) == SUCCESS) { - metrics.estimated_inline_len = estimate_number_length(val); - } - break; - } - case dom::element_type::BIGINT: { - std::string_view digits; - if (elem.get_bigint().get(digits) == SUCCESS) { - metrics.estimated_inline_len = digits.size(); - } - break; - } - case dom::element_type::BOOL: { - bool val; - if (elem.get_bool().get(val) == SUCCESS) { - metrics.estimated_inline_len = val ? 4 : 5; // "true" or "false" - } - break; - } - case dom::element_type::NULL_VALUE: - metrics.estimated_inline_len = 4; // "null" - break; - default: - break; - } - - return metrics; -} - -inline element_metrics structure_analyzer::analyze_array(const dom::array& arr, - size_t depth) const { - element_metrics metrics; - metrics.complexity = 1; // At least 1 for being an array - metrics.estimated_inline_len = 2; // "[]" - metrics.child_count = 0; - - size_t max_child_complexity = 0; - bool first = true; - - for (dom::element child : arr) { - if (!first) { - metrics.estimated_inline_len += current_opts_->comma_padding ? 2 : 1; // ", " or "," - } - first = false; - - element_metrics child_metrics = analyze_element(child, depth + 1); - metrics.estimated_inline_len += child_metrics.estimated_inline_len; - max_child_complexity = (std::max)(max_child_complexity, child_metrics.complexity); - metrics.child_count++; - metrics.children.push_back(std::move(child_metrics)); - } - - // Complexity is 1 + max child complexity - metrics.complexity = 1 + max_child_complexity; - - // Bracket padding "[ 1, 2 ]" vs "[1, 2]" - bool use_bracket_padding = (max_child_complexity >= 1) - ? current_opts_->nested_bracket_padding : current_opts_->simple_bracket_padding; - if (use_bracket_padding && metrics.child_count > 0) { - metrics.estimated_inline_len += 2; - } - - // Check if can inline - metrics.can_inline = metrics.complexity <= current_opts_->max_inline_complexity; - - // Check for uniform array (table formatting, or aligned compact multiline). - bool wants_table_columns = - (current_opts_->enable_table_format && - max_child_complexity <= current_opts_->max_table_row_complexity) || - (current_opts_->enable_compact_multiline && - max_child_complexity <= current_opts_->max_compact_array_complexity); - if (wants_table_columns) { - metrics.is_uniform_array = check_array_uniformity(arr, metrics, depth); - } - - return metrics; -} - -inline element_metrics structure_analyzer::analyze_object(const dom::object& obj, - size_t depth) const { - element_metrics metrics; - metrics.complexity = 1; - metrics.estimated_inline_len = 2; // "{}" - metrics.child_count = 0; - - size_t max_child_complexity = 0; - bool first = true; - - for (dom::key_value_pair field : obj) { - if (!first) { - metrics.estimated_inline_len += current_opts_->comma_padding ? 2 : 1; // ", " or "," - } - first = false; - - // Key length: quotes + key + colon (+ space) - metrics.estimated_inline_len += estimate_string_length(field.key) + - (current_opts_->colon_padding ? 2 : 1); - - element_metrics child_metrics = analyze_element(field.value, depth + 1); - metrics.estimated_inline_len += child_metrics.estimated_inline_len; - max_child_complexity = (std::max)(max_child_complexity, child_metrics.complexity); - metrics.child_count++; - metrics.children.push_back(std::move(child_metrics)); - } - - metrics.complexity = 1 + max_child_complexity; - - // Bracket padding '{ "a": 1 }' vs '{"a": 1}' - bool use_bracket_padding = (max_child_complexity >= 1) - ? current_opts_->nested_bracket_padding : current_opts_->simple_bracket_padding; - if (use_bracket_padding && metrics.child_count > 0) { - metrics.estimated_inline_len += 2; - } - - metrics.can_inline = metrics.complexity <= current_opts_->max_inline_complexity; - - return metrics; -} - -inline size_t structure_analyzer::estimate_string_length(std::string_view s) const { - size_t len = 2; // quotes - for (char c : s) { - unsigned char uc = static_cast(c); - if (c == '"' || c == '\\') { - len += 2; // \" and \\ are two-character escapes - } else if (uc < 0x20) { - switch (c) { - case '\b': case '\f': case '\n': case '\r': case '\t': - len += 2; // two-character escapes: \b \f \n \r \t - break; - default: - len += 6; // \u00XX -- six-character escape for other control chars - break; - } - } else { - len += 1; - } - } - return len; -} - -inline size_t structure_analyzer::estimate_number_length(double d) const { - if (!std::isfinite(d)) { -#if SIMDJSON_ENABLE_NAN_INF - if (std::isnan(d)) { - return 3; // "NaN" - } else if (d < 0) { - return 9; // "-Infinity" - } else { - return 8; // "Infinity" - } -#else - return 4; // "null" for invalid numbers -#endif - } - // Must match base_formatter::number(double), which serializes finite values - // with internal::to_chars (shortest round-trip). snprintf("%.17g") over-counts - // (e.g. 0.1 becomes "0.10000000000000001", 19 chars, vs the emitted "0.1"), - // which misaligns double columns in table formatting the same way escaped - // control characters did before #2929. - char buf[simdjson::internal::to_chars_buffer_size]; - char *end = simdjson::internal::to_chars(buf, buf + sizeof(buf), d); - return static_cast(end - buf); -} - -inline size_t structure_analyzer::estimate_number_length(int64_t i) const { - if (i == 0) return 1; - // Handle INT64_MIN specially to avoid overflow when negating - if (i == INT64_MIN) return 20; // "-9223372036854775808" is 20 characters - size_t len = (i < 0) ? 1 : 0; // negative sign - int64_t abs_val = (i < 0) ? -i : i; - while (abs_val > 0) { - len++; - abs_val /= 10; - } - return len; -} - -inline size_t structure_analyzer::estimate_number_length(uint64_t u) const { - if (u == 0) return 1; - size_t len = 0; - while (u > 0) { - len++; - u /= 10; - } - return len; -} - -inline table_column_type structure_analyzer::classify_table_value(dom::element_type type) { - switch (type) { - case dom::element_type::OBJECT: return table_column_type::object; - case dom::element_type::ARRAY: return table_column_type::array; - case dom::element_type::INT64: - case dom::element_type::UINT64: - case dom::element_type::DOUBLE: - case dom::element_type::BIGINT: return table_column_type::number; - case dom::element_type::NULL_VALUE: return table_column_type::unknown; - default: return table_column_type::simple; // string, bool - } -} - -inline void structure_analyzer::classify_and_measure( - const std::vector>& values, - table_column_type& common, size_t& max_width) { - common = table_column_type::unknown; - max_width = 0; - for (const auto& v : values) { - table_column_type t = classify_table_value(v.first.type()); - if (t != table_column_type::unknown) { - if (common == table_column_type::unknown) common = t; - else if (t != common) common = table_column_type::mixed; - } - if (v.second) { - max_width = (std::max)(max_width, v.second->estimated_inline_len); - } - } -} - -inline void structure_analyzer::build_table_columns( - const std::vector>& values, - std::vector& out_columns) const { - out_columns.clear(); - if (values.empty()) { - return; - } - - table_column_type common = table_column_type::unknown; - for (const auto& v : values) { - table_column_type t = classify_table_value(v.first.type()); - // Null cannot be projected into object or array columns. Keep these - // values intact by falling back to ordinary value formatting. - if (t == table_column_type::unknown) return; - if (common == table_column_type::unknown) common = t; - else if (t != common) { common = table_column_type::mixed; break; } - } - if (common != table_column_type::object && common != table_column_type::array) { - return; - } - - std::vector>> per_column_values; - - if (common == table_column_type::object) { - std::unordered_map column_index; - // Last row (1-based) that filled each column, to detect duplicate keys. - std::vector column_last_row; - size_t row = 0; - for (const auto& v : values) { - if (v.first.type() != dom::element_type::OBJECT) continue; - dom::object obj; - if (v.first.get_object().get(obj) != SUCCESS) continue; - - row++; - size_t field_idx = 0; - for (dom::key_value_pair field : obj) { - auto it = column_index.find(field.key); - size_t col_idx; - if (it == column_index.end()) { - col_idx = out_columns.size(); - column_index.emplace(field.key, col_idx); - out_columns.emplace_back(); - out_columns.back().key.assign(field.key.data(), field.key.size()); - out_columns.back().has_key = true; - out_columns.back().key_width = estimate_string_length(field.key); - per_column_values.emplace_back(); - column_last_row.push_back(0); - } else { - col_idx = it->second; - } - // A row with a duplicate key cannot be laid out as a table: each - // column holds one value per row, so the later duplicates would be - // lost. Returning no columns also disables the aligned compact - // multiline layout for this array, which needs the same columns. - if (column_last_row[col_idx] == row) { - out_columns.clear(); - return; - } - column_last_row[col_idx] = row; - const element_metrics* field_metrics = (v.second && field_idx < v.second->children.size()) - ? &v.second->children[field_idx] : nullptr; - per_column_values[col_idx].emplace_back(field.value, field_metrics); - field_idx++; - } - } - } else { // array: columns by position - for (const auto& v : values) { - if (v.first.type() != dom::element_type::ARRAY) continue; - dom::array sub_arr; - if (v.first.get_array().get(sub_arr) != SUCCESS) continue; - - size_t idx = 0; - for (dom::element item : sub_arr) { - if (out_columns.size() <= idx) { - out_columns.emplace_back(); - per_column_values.emplace_back(); - } - const element_metrics* item_metrics = (v.second && idx < v.second->children.size()) - ? &v.second->children[idx] : nullptr; - per_column_values[idx].emplace_back(item, item_metrics); - idx++; - } - } - } - - for (size_t i = 0; i < out_columns.size(); i++) { - table_column_type col_type; - size_t max_width; - classify_and_measure(per_column_values[i], col_type, max_width); - out_columns[i].type = col_type; - out_columns[i].plain_width = max_width; - - if (col_type == table_column_type::object || col_type == table_column_type::array) { - build_table_columns(per_column_values[i], out_columns[i].children); - } - - out_columns[i].width = out_columns[i].children.empty() ? max_width : compute_columns_width(out_columns[i].children); - } -} - -inline bool structure_analyzer::check_array_uniformity(const dom::array& arr, - element_metrics& metrics, - size_t depth) const { - std::vector> values; - values.reserve(metrics.child_count); - - size_t row_idx = 0; - for (dom::element elem : arr) { - const element_metrics* row_metrics = (row_idx < metrics.children.size()) ? &metrics.children[row_idx] : nullptr; - values.emplace_back(elem, row_metrics); - row_idx++; - } - - build_table_columns(values, metrics.table_columns); - if (metrics.table_columns.empty()) { - // Not uniformly object or array. Check for uniform scalar - table_column_type common; - size_t max_width; - classify_and_measure(values, common, max_width); - if (common != table_column_type::number && common != table_column_type::simple) { - return false; - } - metrics.scalar_column_type = common; - metrics.table_row_width = max_width; - metrics.table_row_width_full = max_width; - return true; - } - - metrics.table_row_width_full = compute_columns_width(metrics.table_columns); - - size_t row_indent_width = (depth + 1) * current_opts_->indent_spaces; - size_t budget = (row_indent_width + 1 >= current_opts_->max_total_line_length) - ? 0 : current_opts_->max_total_line_length - row_indent_width - 1; - - size_t width = metrics.table_row_width_full; - while (width > budget && flatten_deepest_columns(metrics.table_columns)) { - recompute_column_widths(metrics.table_columns); - width = compute_columns_width(metrics.table_columns); - } - - metrics.table_row_width = width; - return true; -} - -inline size_t structure_analyzer::compute_columns_width(const std::vector& columns) const { - size_t width = 2; // "{}" or "[]" - if (table_row_is_nested(columns) ? current_opts_->nested_bracket_padding : current_opts_->simple_bracket_padding) { - width += 2; - } - - for (const table_column& col : columns) { - if (col.has_key) { - width += col.key_width; - width += current_opts_->colon_padding ? 2 : 1; - } - width += col.width; - } - if (columns.size() > 1) { - width += (columns.size() - 1) * (current_opts_->comma_padding ? 2 : 1); - } - return width; -} - -inline size_t structure_analyzer::column_height(const table_column& column) { - size_t height = 0; - for (const table_column& child : column.children) { - height = (std::max)(height, column_height(child)); - } - return column.children.empty() ? 0 : height + 1; -} - -inline bool structure_analyzer::flatten_deepest_columns(std::vector& columns) { - size_t max_height = 0; - for (const table_column& col : columns) { - max_height = (std::max)(max_height, column_height(col)); - } - - bool changed = false; - for (table_column& col : columns) { - if (column_height(col) != max_height || max_height == 0) continue; - if (max_height == 1) { - col.children.clear(); - col.width = col.plain_width; - changed = true; - } else { - changed |= flatten_deepest_columns(col.children); - } - } - return changed; -} - -inline void structure_analyzer::recompute_column_widths(std::vector& columns) const { - for (table_column& col : columns) { - if (!col.children.empty()) { - recompute_column_widths(col.children); - col.width = compute_columns_width(col.children); - } - } -} - -inline layout_mode structure_analyzer::decide_layout(const element_metrics& metrics, - size_t depth, - const fractured_json_options& opts, - bool has_trailing_comma, - size_t key_width) { - if (metrics.child_count == 0) { - return layout_mode::single_line; - } - - long long signed_depth = static_cast(depth); - bool depth_allows_inline_or_compact = signed_depth > opts.always_expand_depth; - bool depth_allows_table = signed_depth >= opts.always_expand_depth; - - // Check inline feasibility - size_t reserved_width = depth * opts.indent_spaces + key_width + - (has_trailing_comma ? 1 : 0); - if (depth_allows_inline_or_compact && metrics.can_inline && - metrics.estimated_inline_len + reserved_width <= opts.max_total_line_length) { - return layout_mode::single_line; - } - - // Rows (table's or compact multiline's) render one level deeper than the - // array itself. - size_t row_indent_width = (depth + 1) * opts.indent_spaces; - - // Check compact multiline - // for uniform arrays fall back to table if we would have to flatten any formatting - bool compact_multiline_enabled = opts.enable_compact_multiline && - metrics.complexity <= opts.max_compact_array_complexity + 1 && - metrics.child_count >= opts.min_compact_array_row_items; - if (depth_allows_inline_or_compact && compact_multiline_enabled) { - bool aligned = metrics.is_uniform_array; - size_t comma_width = opts.comma_padding ? 2 : 1; - size_t avg_item_width; - if (aligned) { - avg_item_width = metrics.table_row_width_full + comma_width; - } else { - size_t sum = 0; - for (const element_metrics& child : metrics.children) { - sum += child.estimated_inline_len; - } - avg_item_width = comma_width + sum / metrics.child_count; - } - - size_t row_pack_space = (row_indent_width >= opts.max_total_line_length) - ? 0 : opts.max_total_line_length - row_indent_width; - if (avg_item_width * opts.min_compact_array_row_items <= row_pack_space) { - return layout_mode::compact_multiline; - } - } - - // Check Table mode - if (depth_allows_table && opts.enable_table_format && - metrics.is_uniform_array && - metrics.table_row_width + 1 + row_indent_width <= opts.max_total_line_length) { - return layout_mode::table; - } - - return layout_mode::expanded; -} - -// -// Fractured Formatter Implementation -// - -inline fractured_formatter::fractured_formatter(const fractured_json_options& opts) - : options_(opts) {} - -simdjson_inline void fractured_formatter::print_newline() { - if (current_layout_ == layout_mode::single_line) { - return; // No newlines in inline mode - } - // Like the reference FracturedJson implementation, do not end a line with - // white space: the padding that aligns a value in a column is only needed - // when something follows it on the same line. - buffer.trim_trailing_spaces(); - one_char('\n'); - current_line_length_ = 0; -} - -simdjson_inline void fractured_formatter::print_indents(size_t depth) { - if (current_layout_ == layout_mode::single_line) { - return; // No indentation in inline mode - } - for (size_t i = 0; i < depth * options_.indent_spaces; i++) { - one_char(' '); - current_line_length_++; - } -} - -simdjson_inline void fractured_formatter::print_space() { - one_char(' '); - current_line_length_++; -} - -inline void fractured_formatter::set_layout_mode(layout_mode mode) { - current_layout_ = mode; -} - -inline layout_mode fractured_formatter::get_layout_mode() const { - return current_layout_; -} - -inline void fractured_formatter::track_line_length(size_t chars) { - current_line_length_ += chars; -} - -inline bool fractured_formatter::should_break_line(size_t upcoming_length) const { - return (current_line_length_ + upcoming_length) > options_.max_total_line_length; -} - -inline const fractured_json_options& fractured_formatter::options() const { - return options_; -} - -// -// Fractured String Builder Implementation -// - -inline fractured_string_builder::fractured_string_builder(const fractured_json_options& opts) - : format_(opts), analyzer_{}, options_(opts) {} - -inline void fractured_string_builder::append(const dom::element& value) { - // Phase 1: Analyze structure (metrics tree is built recursively) - element_metrics root_metrics = analyzer_.analyze(value, options_); - - // Phase 2: Format using metrics tree (passed through recursion) - format_element(value, root_metrics, 0); -} - -inline void fractured_string_builder::append(const dom::array& value) { - // Analyze the array to get proper metrics with children - element_metrics metrics = analyzer_.analyze_array(value, options_); - format_array(value, metrics, 0); -} - -inline void fractured_string_builder::append(const dom::object& value) { - // Analyze the object to get proper metrics with children - element_metrics metrics = analyzer_.analyze_object(value, options_); - format_object(value, metrics, 0); -} - -simdjson_inline void fractured_string_builder::clear() { - format_.clear(); - analyzer_.clear(); -} - -simdjson_inline std::string_view fractured_string_builder::str() const { - return format_.str(); -} - -inline void fractured_string_builder::format_element(const dom::element& elem, - const element_metrics& metrics, - size_t depth, - bool has_trailing_comma, - size_t key_width) { - switch (elem.type()) { - case dom::element_type::ARRAY: { - dom::array arr; - if (elem.get_array().get(arr) == SUCCESS) { - format_array(arr, metrics, depth, has_trailing_comma, key_width); - } - break; - } - case dom::element_type::OBJECT: { - dom::object obj; - if (elem.get_object().get(obj) == SUCCESS) { - format_object(obj, metrics, depth, has_trailing_comma, key_width); - } - break; - } - default: - format_scalar(elem); - break; - } -} - -inline void fractured_string_builder::format_array(const dom::array& arr, - const element_metrics& metrics, - size_t depth, - bool has_trailing_comma, - size_t key_width) { - layout_mode layout = structure_analyzer::decide_layout(metrics, depth, options_, - has_trailing_comma, key_width); - switch (layout) { - case layout_mode::single_line: - format_array_inline(arr, metrics); - break; - case layout_mode::compact_multiline: - format_array_compact_multiline(arr, metrics, depth); - break; - case layout_mode::table: - format_array_as_table(arr, metrics, depth); - break; - case layout_mode::expanded: - default: - format_array_expanded(arr, metrics, depth); - break; - } -} - -// The children of an inline value are written inline as well. Going through -// format_element() would let decide_layout() pick another layout for them -// (e.g. at a depth not past always_expand_depth), which a single line cannot -// show: it would only add table padding and drop the comma padding. -inline void fractured_string_builder::format_element_inline(const dom::element& elem, - const element_metrics& metrics) { - switch (elem.type()) { - case dom::element_type::ARRAY: { - dom::array arr; - if (elem.get_array().get(arr) == SUCCESS) { - format_array_inline(arr, metrics); - } - break; - } - case dom::element_type::OBJECT: { - dom::object obj; - if (elem.get_object().get(obj) == SUCCESS) { - format_object_inline(obj, metrics); - } - break; - } - default: - format_scalar(elem); - break; - } -} - -inline void fractured_string_builder::format_array_inline(const dom::array& arr, - const element_metrics& metrics) { - scoped_single_line_mode single_line(format_); - - format_.start_array(); - - bool first = true; - bool empty = true; - size_t child_idx = 0; - for (dom::element elem : arr) { - empty = false; - if (!first) { - format_.comma(); - if (options_.comma_padding) { - format_.print_space(); - } - } else if (bracket_padding_for(metrics)) { - format_.print_space(); - } - first = false; - const element_metrics& child_metrics = child_metrics_at(metrics.children, child_idx); - format_element_inline(elem, child_metrics); - child_idx++; - } - - if (bracket_padding_for(metrics) && !empty) { - format_.print_space(); - } - format_.end_array(); -} - -inline void fractured_string_builder::format_array_compact_multiline(const dom::array& arr, - const element_metrics& metrics, - size_t depth) { - if (metrics.is_uniform_array) { - format_array_compact_multiline_aligned(arr, metrics, depth); - return; - } - - format_.start_array(); - format_.print_newline(); - format_.print_indents(depth + 1); - - bool first = true; - bool prev_item_was_expanded = false; - size_t child_idx = 0; - - for (dom::element elem : arr) { - const element_metrics& child_metrics = child_metrics_at(metrics.children, child_idx); - bool is_last = (child_idx + 1 == metrics.child_count); - - if (!first) { - format_.comma(); - format_.track_line_length(1); - - // Check if we should break to new line. Staying on this line also - // takes the padding space before the item and the comma after it. - size_t upcoming_length = child_metrics.estimated_inline_len + - (options_.comma_padding ? 1 : 0) + - (is_last ? 0 : 1); - if (prev_item_was_expanded || - format_.should_break_line(upcoming_length)) { - format_.print_newline(); - format_.print_indents(depth + 1); - } else if (options_.comma_padding) { - format_.print_space(); - } - } - first = false; - - layout_mode item_layout = structure_analyzer::decide_layout(child_metrics, depth + 1, options_, !is_last); - bool item_fits = item_layout == layout_mode::single_line; - if (item_fits) { - { - scoped_single_line_mode single_line(format_); - format_element(elem, child_metrics, depth + 1, !is_last); - } - format_.track_line_length(child_metrics.estimated_inline_len); - } else { - format_element(elem, child_metrics, depth + 1, !is_last); - } - prev_item_was_expanded = !item_fits; - - child_idx++; - } - - format_.print_newline(); - format_.print_indents(depth); - format_.end_array(); -} - -inline void fractured_string_builder::format_array_compact_multiline_aligned( - const dom::array& arr, const element_metrics& metrics, size_t depth) { - const std::vector& columns = metrics.table_columns; - - format_.start_array(); - format_.print_newline(); - format_.print_indents(depth + 1); - - size_t indent_width = (depth + 1) * options_.indent_spaces; - size_t available_line_space = (indent_width >= options_.max_total_line_length) - ? 0 : options_.max_total_line_length - indent_width; - size_t comma_width = options_.comma_padding ? 2 : 1; - size_t remaining_line_space = available_line_space; - - bool first = true; - size_t child_idx = 0; - - for (dom::element elem : arr) { - bool needs_comma = (child_idx + 1 < metrics.child_count); - size_t space_needed = metrics.table_row_width_full + (needs_comma ? comma_width : 0); - - if (!first) { - if (remaining_line_space < space_needed) { - format_.print_newline(); - format_.print_indents(depth + 1); - remaining_line_space = available_line_space; - } else if (options_.comma_padding) { - format_.print_space(); - } - } - first = false; - - const element_metrics& row_metrics = child_metrics_at(metrics.children, child_idx); - if (columns.empty()) { - format_table_scalar_row(elem, row_metrics, metrics.table_row_width_full, - metrics.scalar_column_type); - } else { - format_table_row(elem, row_metrics, columns, depth + 1); - } - if (needs_comma) { - format_.comma(); - } - remaining_line_space -= (std::min)(remaining_line_space, space_needed); - child_idx++; - } - - format_.print_newline(); - format_.print_indents(depth); - format_.end_array(); -} - -inline void fractured_string_builder::format_table_row_columns( - const std::vector& columns, - const std::vector& found, - const std::vector& values, - const std::vector& value_metrics, - size_t depth) { - const size_t num_columns = columns.size(); - size_t last_present_idx = num_columns; - for (size_t i = 0; i < num_columns; i++) { - if (found[i]) last_present_idx = i; - } - - size_t comma_width = options_.comma_padding ? 2 : 1; - - for (size_t col_idx = 0; col_idx < num_columns; col_idx++) { - const table_column& column = columns[col_idx]; - const bool is_last_col = (col_idx == num_columns - 1); - - if (found[col_idx]) { - if (column.has_key) { - format_.key(column.key); - if (options_.colon_padding) { - format_.print_space(); - } - } - - bool needs_comma = !is_last_col && (col_idx < last_present_idx); - - if (!column.children.empty()) { - // Recurses into this cell's own columns instead of a plain value; - // every row aligns those the same way (blank-padding missing - // ones), so the result is always exactly column.width wide - // no padding needed afterward, unlike the leaf case below. - if (column.type == table_column_type::object) { - dom::object sub_obj; - if (values[col_idx].get_object().get(sub_obj) == SUCCESS) { - const element_metrics& sub_metrics = child_metrics_at(value_metrics[col_idx]); - format_table_object_row(sub_obj, sub_metrics, column.children, depth); - } - } else { - dom::array sub_arr; - if (values[col_idx].get_array().get(sub_arr) == SUCCESS) { - const element_metrics& sub_metrics = child_metrics_at(value_metrics[col_idx]); - format_table_array_row(sub_arr, sub_metrics, column.children, depth); - } - } - // value is already padded - if (needs_comma) { - format_.comma(); - if (options_.comma_padding) { - format_.print_space(); - } - } - } else { - const element_metrics& vm = child_metrics_at(value_metrics[col_idx]); - format_table_leaf_value(values[col_idx], vm, column.width, column.type, needs_comma, - /*add_comma_space=*/true); - } - - if (!is_last_col && !needs_comma) { - // Found, but no more real values follow: blank space where a comma would go. - for (size_t i = 0; i < comma_width; i++) { - format_.one_char(' '); - } - } - } else { - size_t slot_width = column.width; - if (column.has_key) { - slot_width += column.key_width + (options_.colon_padding ? 2 : 1); - } - for (size_t i = 0; i < slot_width; i++) { - format_.one_char(' '); - } - - if (!is_last_col) { - for (size_t i = 0; i < comma_width; i++) { - format_.one_char(' '); - } - } - } - } -} - -inline void fractured_string_builder::format_table_object_row( - const dom::object& obj, const element_metrics& row_metrics, - const std::vector& columns, size_t depth) { - const size_t num_columns = columns.size(); - std::vector found(num_columns, false); - std::vector values(num_columns); - std::vector value_metrics(num_columns, nullptr); - - for (size_t col_idx = 0; col_idx < num_columns; col_idx++) { - size_t field_idx = 0; - for (dom::key_value_pair field : obj) { - if (field.key == columns[col_idx].key) { - found[col_idx] = true; - values[col_idx] = field.value; - value_metrics[col_idx] = (field_idx < row_metrics.children.size()) - ? &row_metrics.children[field_idx] : nullptr; - break; - } - field_idx++; - } - } - - bool nested = table_row_is_nested(columns); - format_.start_object(); - if (nested ? options_.nested_bracket_padding : options_.simple_bracket_padding) { - format_.print_space(); - } - format_table_row_columns(columns, found, values, value_metrics, depth); - if (nested ? options_.nested_bracket_padding : options_.simple_bracket_padding) { - format_.print_space(); - } - format_.end_object(); -} - -inline void fractured_string_builder::format_table_array_row( - const dom::array& arr, const element_metrics& row_metrics, - const std::vector& columns, size_t depth) { - const size_t num_columns = columns.size(); - std::vector found(num_columns, false); - std::vector values(num_columns); - std::vector value_metrics(num_columns, nullptr); - - size_t idx = 0; - for (dom::element item : arr) { - if (idx >= num_columns) break; - found[idx] = true; - values[idx] = item; - value_metrics[idx] = (idx < row_metrics.children.size()) ? &row_metrics.children[idx] : nullptr; - idx++; - } - - bool nested = table_row_is_nested(columns); - format_.start_array(); - if (nested ? options_.nested_bracket_padding : options_.simple_bracket_padding) { - format_.print_space(); - } - format_table_row_columns(columns, found, values, value_metrics, depth); - if (nested ? options_.nested_bracket_padding : options_.simple_bracket_padding) { - format_.print_space(); - } - format_.end_array(); -} - -inline void fractured_string_builder::format_table_row( - const dom::element& elem, const element_metrics& row_metrics, - const std::vector& columns, size_t depth) { - if (elem.type() == dom::element_type::ARRAY) { - dom::array arr; - if (elem.get_array().get(arr) == SUCCESS) { - format_table_array_row(arr, row_metrics, columns, depth); - } - } else { - dom::object obj; - if (elem.get_object().get(obj) == SUCCESS) { - format_table_object_row(obj, row_metrics, columns, depth); - } - } -} - -inline bool fractured_string_builder::comma_goes_before_padding(table_column_type column_type) const { - switch (options_.comma_placement) { - case table_comma_placement::before_padding: return true; - case table_comma_placement::after_padding: return false; - case table_comma_placement::before_padding_except_numbers: - default: - return column_type != table_column_type::number; - } -} - -inline void fractured_string_builder::format_table_leaf_value( - const dom::element& elem, const element_metrics& vm, size_t width, - table_column_type column_type, bool needs_comma, bool add_comma_space) { - bool comma_before_pad = needs_comma && comma_goes_before_padding(column_type); - bool comma_after_pad = needs_comma && !comma_before_pad; - - bool right_align = column_type == table_column_type::number && - options_.number_alignment == number_list_alignment::right; - - size_t value_len = vm.estimated_inline_len; - size_t left_pad = 0; - size_t right_pad = 0; - if (right_align) { - left_pad = (width > value_len) ? width - value_len : 0; - comma_before_pad = needs_comma; - comma_after_pad = false; - } else { - right_pad = (width > value_len) ? width - value_len : 0; - } - - for (size_t i = 0; i < left_pad; i++) { - format_.one_char(' '); - } - - // The cell shares its line with the rest of the row. - format_element_inline(elem, vm); - - if (comma_before_pad) { - format_.comma(); - } - for (size_t i = 0; i < right_pad; i++) { - format_.one_char(' '); - } - if (comma_after_pad) { - format_.comma(); - } - if (needs_comma && add_comma_space && options_.comma_padding) { - format_.print_space(); - } -} - -inline void fractured_string_builder::format_table_scalar_row( - const dom::element& elem, const element_metrics& row_metrics, size_t width, - table_column_type column_type) { - format_table_leaf_value(elem, row_metrics, width, column_type, - /*needs_comma=*/false, /*add_comma_space=*/false); -} - -inline void fractured_string_builder::format_array_as_table(const dom::array& arr, - const element_metrics& metrics, - size_t depth) { - if (!metrics.is_uniform_array) { - format_array_expanded(arr, metrics, depth); - return; - } - const std::vector& columns = metrics.table_columns; - - format_.start_array(); - format_.print_newline(); - - bool first_row = true; - size_t child_idx = 0; - for (dom::element elem : arr) { - if (!first_row) { - format_.comma(); - format_.print_newline(); - } - first_row = false; - - format_.print_indents(depth + 1); - - const element_metrics& row_metrics = child_metrics_at(metrics.children, child_idx); - if (columns.empty()) { - format_table_scalar_row(elem, row_metrics, metrics.table_row_width, - metrics.scalar_column_type); - } else { - format_table_row(elem, row_metrics, columns, depth + 1); - } - child_idx++; - } - - format_.print_newline(); - format_.print_indents(depth); - format_.end_array(); -} - -inline void fractured_string_builder::format_array_expanded(const dom::array& arr, - const element_metrics& metrics, - size_t depth) { - format_.start_array(); - - bool empty = true; - bool first = true; - size_t child_idx = 0; - - for (dom::element elem : arr) { - empty = false; - if (!first) { - format_.comma(); - } - first = false; - - format_.print_newline(); - format_.print_indents(depth + 1); - const element_metrics& child_metrics = child_metrics_at(metrics.children, child_idx); - bool is_last = (child_idx + 1 == metrics.child_count); - format_element(elem, child_metrics, depth + 1, !is_last); - child_idx++; - } - - if (!empty) { - format_.print_newline(); - format_.print_indents(depth); - } - format_.end_array(); -} - -inline void fractured_string_builder::format_object(const dom::object& obj, - const element_metrics& metrics, - size_t depth, - bool has_trailing_comma, - size_t key_width) { - layout_mode layout = structure_analyzer::decide_layout(metrics, depth, options_, - has_trailing_comma, key_width); - if (layout == layout_mode::single_line) { - format_object_inline(obj, metrics); - } else { - format_object_expanded(obj, metrics, depth); - } -} - -inline void fractured_string_builder::format_object_inline(const dom::object& obj, - const element_metrics& metrics) { - scoped_single_line_mode single_line(format_); - - format_.start_object(); - - bool empty = true; - bool first = true; - size_t child_idx = 0; - - for (dom::key_value_pair field : obj) { - empty = false; - if (!first) { - format_.comma(); - if (options_.comma_padding) { - format_.print_space(); - } - } else if (bracket_padding_for(metrics)) { - format_.print_space(); - } - first = false; - - format_.key(field.key); - if (options_.colon_padding) { - format_.print_space(); - } - const element_metrics& child_metrics = child_metrics_at(metrics.children, child_idx); - format_element_inline(field.value, child_metrics); - child_idx++; - } - - if (bracket_padding_for(metrics) && !empty) { - format_.print_space(); - } - format_.end_object(); -} - -inline void fractured_string_builder::format_object_expanded(const dom::object& obj, - const element_metrics& metrics, - size_t depth) { - format_.start_object(); - - bool empty = true; - bool first = true; - size_t child_idx = 0; - - for (dom::key_value_pair field : obj) { - empty = false; - if (!first) { - format_.comma(); - } - first = false; - - format_.print_newline(); - format_.print_indents(depth + 1); - format_.key(field.key); - if (options_.colon_padding) { - format_.print_space(); - } - const element_metrics& child_metrics = child_metrics_at(metrics.children, child_idx); - bool is_last = (child_idx + 1 == metrics.child_count); - // An inline value shares its line with the key, the colon and its padding. - size_t key_width = analyzer_.estimate_string_length(field.key) + 1 + - (options_.colon_padding ? 1 : 0); - format_element(field.value, child_metrics, depth + 1, !is_last, key_width); - child_idx++; - } - - if (!empty) { - format_.print_newline(); - format_.print_indents(depth); - } - format_.end_object(); -} - -inline void fractured_string_builder::format_scalar(const dom::element& elem) { - switch (elem.type()) { - case dom::element_type::STRING: { - std::string_view str; - if (elem.get_string().get(str) == SUCCESS) { - format_.string(str); - } - break; - } - case dom::element_type::INT64: { - int64_t val; - if (elem.get_int64().get(val) == SUCCESS) { - format_.number(val); - } - break; - } - case dom::element_type::UINT64: { - uint64_t val; - if (elem.get_uint64().get(val) == SUCCESS) { - format_.number(val); - } - break; - } - case dom::element_type::DOUBLE: { - double val; - if (elem.get_double().get(val) == SUCCESS) { - format_.number(val); - } - break; - } - case dom::element_type::BIGINT: { - // Big integer stored as string: output the raw digits (no quotes) - std::string_view digits; - if (elem.get_bigint().get(digits) == SUCCESS) { - format_.chars(digits.data(), digits.data() + digits.size()); - } - break; - } - case dom::element_type::BOOL: { - bool val; - if (elem.get_bool().get(val) == SUCCESS) { - val ? format_.true_atom() : format_.false_atom(); - } - break; - } - case dom::element_type::NULL_VALUE: - format_.null_atom(); - break; - default: - break; - } -} - -inline bool fractured_string_builder::bracket_padding_for(const element_metrics& metrics) const { - return metrics.complexity >= 2 ? options_.nested_bracket_padding : options_.simple_bracket_padding; -} - -} // namespace internal - -// -// Public API Implementation -// - -template -std::string fractured_json(T x) { - return fractured_json(x, fractured_json_options{}); -} - -template -std::string fractured_json(T x, const fractured_json_options& options) { - internal::fractured_string_builder sb(options); - sb.append(x); - std::string_view result = sb.str(); - return std::string(result.data(), result.size()); -} - -#if SIMDJSON_EXCEPTIONS -template -std::string fractured_json(simdjson_result x) { - if (x.error()) { - throw simdjson_error(x.error()); - } - return fractured_json(x.value()); -} - -template -std::string fractured_json(simdjson_result x, const fractured_json_options& options) { - if (x.error()) { - throw simdjson_error(x.error()); - } - return fractured_json(x.value(), options); -} -#endif - -// -// String-based API for formatting any JSON string -// - -inline std::string fractured_json_string(std::string_view json_str) { - return fractured_json_string(json_str, fractured_json_options{}); -} - -inline std::string fractured_json_string(std::string_view json_str, - const fractured_json_options& options) { - // Parse the JSON string - dom::parser parser; - dom::element doc; - // Need to pad the string for simdjson - auto padded = padded_string(json_str); - auto error = parser.parse(padded).get(doc); - if (error) { - // If parsing fails, return the original string - return std::string(json_str); - } - return fractured_json(doc, options); -} - -} // namespace simdjson - -#endif // SIMDJSON_DOM_FRACTURED_JSON_INL_H -/* end file simdjson/dom/fractured_json-inl.h */ - -#endif // SIMDJSON_DOM_H -/* end file simdjson/dom.h */ -/* including simdjson/builder.h: #include "simdjson/builder.h" */ -/* begin file simdjson/builder.h */ -#ifndef SIMDJSON_BUILDER_H -#define SIMDJSON_BUILDER_H - -/* including simdjson/builtin/builder.h: #include "simdjson/builtin/builder.h" */ -/* begin file simdjson/builtin/builder.h */ -#ifndef SIMDJSON_BUILTIN_BUILDER_H -#define SIMDJSON_BUILTIN_BUILDER_H - -/* including simdjson/builtin.h: #include "simdjson/builtin.h" */ -/* begin file simdjson/builtin.h */ -#ifndef SIMDJSON_BUILTIN_H -#define SIMDJSON_BUILTIN_H - -/* including simdjson/builtin/base.h: #include "simdjson/builtin/base.h" */ -/* begin file simdjson/builtin/base.h */ -#ifndef SIMDJSON_BUILTIN_BASE_H -#define SIMDJSON_BUILTIN_BASE_H - -/* skipped duplicate #include "simdjson/base.h" */ -/* including simdjson/implementation_detection.h: #include "simdjson/implementation_detection.h" */ -/* begin file simdjson/implementation_detection.h */ -#ifndef SIMDJSON_IMPLEMENTATION_DETECTION_H -#define SIMDJSON_IMPLEMENTATION_DETECTION_H - -/* skipped duplicate #include "simdjson/base.h" */ - -// 0 is reserved, because undefined SIMDJSON_IMPLEMENTATION equals 0 in preprocessor macros. -#define SIMDJSON_IMPLEMENTATION_ID_arm64 1 -#define SIMDJSON_IMPLEMENTATION_ID_fallback 2 -#define SIMDJSON_IMPLEMENTATION_ID_haswell 3 -#define SIMDJSON_IMPLEMENTATION_ID_icelake 4 -#define SIMDJSON_IMPLEMENTATION_ID_ppc64 5 -#define SIMDJSON_IMPLEMENTATION_ID_westmere 6 -#define SIMDJSON_IMPLEMENTATION_ID_lsx 7 -#define SIMDJSON_IMPLEMENTATION_ID_lasx 8 -//#define SIMDJSON_IMPLEMENTATION_ID_rvv 9 -#define SIMDJSON_IMPLEMENTATION_ID_rvv_vls 10 - -#define SIMDJSON_IMPLEMENTATION_ID_FOR(IMPL) SIMDJSON_CAT(SIMDJSON_IMPLEMENTATION_ID_, IMPL) -#define SIMDJSON_IMPLEMENTATION_ID SIMDJSON_IMPLEMENTATION_ID_FOR(SIMDJSON_IMPLEMENTATION) - -#define SIMDJSON_IMPLEMENTATION_IS(IMPL) SIMDJSON_IMPLEMENTATION_ID == SIMDJSON_IMPLEMENTATION_ID_FOR(IMPL) - -// -// First, figure out which implementations can be run. Doing it here makes it so we don't have to worry about the order -// in which we include them. -// - -#ifndef SIMDJSON_IMPLEMENTATION_ARM64 -#define SIMDJSON_IMPLEMENTATION_ARM64 (SIMDJSON_IS_ARM64) -#endif -#if SIMDJSON_IMPLEMENTATION_ARM64 && SIMDJSON_IS_ARM64 -#define SIMDJSON_CAN_ALWAYS_RUN_ARM64 1 -#else -#define SIMDJSON_CAN_ALWAYS_RUN_ARM64 0 -#endif - -// Default Icelake to on if this is x86-64. Even if we're not compiled for it, it could be selected -// at runtime. -#ifndef SIMDJSON_IMPLEMENTATION_ICELAKE -#define SIMDJSON_IMPLEMENTATION_ICELAKE ((SIMDJSON_IS_X86_64) && (SIMDJSON_AVX512_ALLOWED) && (SIMDJSON_COMPILER_SUPPORTS_VBMI2)) -#endif - -#ifdef _MSC_VER -// To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this next line, see -// https://github.com/simdjson/simdjson/issues/1247 -#if ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__)) -#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 1 -#else -#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 0 -#endif - -#else - -#if ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__)) -#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 1 -#else -#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 0 -#endif - -#endif - -// Default Haswell to on if this is x86-64. Even if we're not compiled for it, it could be selected -// at runtime. -#ifndef SIMDJSON_IMPLEMENTATION_HASWELL -#if SIMDJSON_CAN_ALWAYS_RUN_ICELAKE -// if icelake is always available, never enable haswell. -#define SIMDJSON_IMPLEMENTATION_HASWELL 0 -#else -#define SIMDJSON_IMPLEMENTATION_HASWELL SIMDJSON_IS_X86_64 -#endif -#endif -#ifdef _MSC_VER -// To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this next line, see -// https://github.com/simdjson/simdjson/issues/1247 -#if ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__)) -#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 1 -#else -#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 0 -#endif - -#else - -#if ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__)) -#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 1 -#else -#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 0 -#endif - -#endif - -// Default Westmere to on if this is x86-64. -#ifndef SIMDJSON_IMPLEMENTATION_WESTMERE -#if SIMDJSON_CAN_ALWAYS_RUN_ICELAKE || SIMDJSON_CAN_ALWAYS_RUN_HASWELL -// if icelake or haswell are always available, never enable westmere. -#define SIMDJSON_IMPLEMENTATION_WESTMERE 0 -#else -#define SIMDJSON_IMPLEMENTATION_WESTMERE SIMDJSON_IS_X86_64 -#endif -#endif - -#if (SIMDJSON_IMPLEMENTATION_WESTMERE && SIMDJSON_IS_X86_64 && __SSE4_2__ && __PCLMUL__) -#define SIMDJSON_CAN_ALWAYS_RUN_WESTMERE 1 -#else -#define SIMDJSON_CAN_ALWAYS_RUN_WESTMERE 0 -#endif - - -#ifndef SIMDJSON_IMPLEMENTATION_PPC64 -#define SIMDJSON_IMPLEMENTATION_PPC64 (SIMDJSON_IS_PPC64 && SIMDJSON_IS_PPC64_VMX) -#endif -#if SIMDJSON_IMPLEMENTATION_PPC64 && SIMDJSON_IS_PPC64 && SIMDJSON_IS_PPC64_VMX -#define SIMDJSON_CAN_ALWAYS_RUN_PPC64 1 -#else -#define SIMDJSON_CAN_ALWAYS_RUN_PPC64 0 -#endif - -#ifndef SIMDJSON_IMPLEMENTATION_LASX -#define SIMDJSON_IMPLEMENTATION_LASX (SIMDJSON_IS_LSX) -#endif -#define SIMDJSON_CAN_ALWAYS_RUN_LASX (SIMDJSON_IS_LASX) - -#ifndef SIMDJSON_IMPLEMENTATION_LSX -#if SIMDJSON_CAN_ALWAYS_RUN_LASX -#define SIMDJSON_IMPLEMENTATION_LSX 0 -#else -#define SIMDJSON_IMPLEMENTATION_LSX (SIMDJSON_IS_LSX) -#endif -#endif -#define SIMDJSON_CAN_ALWAYS_RUN_LSX (SIMDJSON_IMPLEMENTATION_LSX) - -#define SIMDJSON_CAN_ALWAYS_RUN_RVV_VLS SIMDJSON_IS_RVV_VLS -#ifndef SIMDJSON_IMPLEMENTATION_RVV_VLS -#define SIMDJSON_IMPLEMENTATION_RVV_VLS SIMDJSON_CAN_ALWAYS_RUN_RVV_VLS -#endif - -// Default Fallback to on unless a builtin implementation has already been selected. -#ifndef SIMDJSON_IMPLEMENTATION_FALLBACK -#if SIMDJSON_CAN_ALWAYS_RUN_ARM64 || SIMDJSON_CAN_ALWAYS_RUN_ICELAKE || SIMDJSON_CAN_ALWAYS_RUN_HASWELL || SIMDJSON_CAN_ALWAYS_RUN_WESTMERE || SIMDJSON_CAN_ALWAYS_RUN_PPC64 || SIMDJSON_CAN_ALWAYS_RUN_LSX || SIMDJSON_CAN_ALWAYS_RUN_LASX || SIMDJSON_CAN_ALWAYS_RUN_RVV_VLS -// if anything at all except fallback can always run, then disable fallback. -#define SIMDJSON_IMPLEMENTATION_FALLBACK 0 -#else -#define SIMDJSON_IMPLEMENTATION_FALLBACK 1 -#endif -#endif -#define SIMDJSON_CAN_ALWAYS_RUN_FALLBACK SIMDJSON_IMPLEMENTATION_FALLBACK - -// Determine the best builtin implementation -#ifndef SIMDJSON_BUILTIN_IMPLEMENTATION - -#if SIMDJSON_CAN_ALWAYS_RUN_ICELAKE -#define SIMDJSON_BUILTIN_IMPLEMENTATION icelake -#elif SIMDJSON_CAN_ALWAYS_RUN_HASWELL -#define SIMDJSON_BUILTIN_IMPLEMENTATION haswell -#elif SIMDJSON_CAN_ALWAYS_RUN_WESTMERE -#define SIMDJSON_BUILTIN_IMPLEMENTATION westmere -#elif SIMDJSON_CAN_ALWAYS_RUN_ARM64 -#define SIMDJSON_BUILTIN_IMPLEMENTATION arm64 -#elif SIMDJSON_CAN_ALWAYS_RUN_PPC64 -#define SIMDJSON_BUILTIN_IMPLEMENTATION ppc64 -#elif SIMDJSON_CAN_ALWAYS_RUN_LSX -#define SIMDJSON_BUILTIN_IMPLEMENTATION lsx -#elif SIMDJSON_CAN_ALWAYS_RUN_LASX -#define SIMDJSON_BUILTIN_IMPLEMENTATION lasx -#elif SIMDJSON_CAN_ALWAYS_RUN_RVV_VLS -#define SIMDJSON_BUILTIN_IMPLEMENTATION rvv_vls -#elif SIMDJSON_CAN_ALWAYS_RUN_FALLBACK -#define SIMDJSON_BUILTIN_IMPLEMENTATION fallback -#else -#error "All possible implementations (including fallback) have been disabled! simdjson will not run." -#endif - -#endif // SIMDJSON_BUILTIN_IMPLEMENTATION - -#define SIMDJSON_BUILTIN_IMPLEMENTATION_ID SIMDJSON_IMPLEMENTATION_ID_FOR(SIMDJSON_BUILTIN_IMPLEMENTATION) -#define SIMDJSON_BUILTIN_IMPLEMENTATION_IS(IMPL) SIMDJSON_BUILTIN_IMPLEMENTATION_ID == SIMDJSON_IMPLEMENTATION_ID_FOR(IMPL) - -#endif // SIMDJSON_IMPLEMENTATION_DETECTION_H -/* end file simdjson/implementation_detection.h */ - -namespace simdjson { -#if SIMDJSON_BUILTIN_IMPLEMENTATION_IS(arm64) - namespace arm64 {} -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(fallback) - namespace fallback {} -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(haswell) - namespace haswell {} -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(icelake) - namespace icelake {} -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(ppc64) - namespace ppc64 {} -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(westmere) - namespace westmere {} -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(lsx) - namespace lsx {} -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(lasx) - namespace lasx {} -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(rvv_vls) - namespace rvv_vls {} -#else -#error Unknown SIMDJSON_BUILTIN_IMPLEMENTATION -#endif - - /** - * Represents the best statically linked simdjson implementation that can be used by the compiling - * program. - * - * Detects what options the program is compiled against, and picks the minimum implementation that - * will work on any computer that can run the program. For example, if you compile with g++ - * -march=westmere, it will pick the westmere implementation. The haswell implementation will - * still be available, and can be selected at runtime, but the builtin implementation (and any - * code that uses it) will use westmere. - */ - namespace builtin = SIMDJSON_BUILTIN_IMPLEMENTATION; -} // namespace simdjson - -#endif // SIMDJSON_BUILTIN_BASE_H -/* end file simdjson/builtin/base.h */ -/* including simdjson/builtin/implementation.h: #include "simdjson/builtin/implementation.h" */ -/* begin file simdjson/builtin/implementation.h */ -#ifndef SIMDJSON_BUILTIN_IMPLEMENTATION_H -#define SIMDJSON_BUILTIN_IMPLEMENTATION_H - -/* skipped duplicate #include "simdjson/builtin/base.h" */ - -/* including simdjson/generic/dependencies.h: #include "simdjson/generic/dependencies.h" */ -/* begin file simdjson/generic/dependencies.h */ -#ifdef SIMDJSON_CONDITIONAL_INCLUDE -#error simdjson/generic/dependencies.h must be included before defining SIMDJSON_CONDITIONAL_INCLUDE! -#endif - -#ifndef SIMDJSON_GENERIC_DEPENDENCIES_H -#define SIMDJSON_GENERIC_DEPENDENCIES_H - -// Internal headers needed for generics. -// All includes referencing simdjson headers *not* under simdjson/generic must be here! -// Otherwise, amalgamation will fail. -/* skipped duplicate #include "simdjson/base.h" */ -/* skipped duplicate #include "simdjson/implementation.h" */ -/* skipped duplicate #include "simdjson/implementation_detection.h" */ -/* including simdjson/internal/instruction_set.h: #include "simdjson/internal/instruction_set.h" */ -/* begin file simdjson/internal/instruction_set.h */ -/* From -https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h -Highly modified. - -Copyright (c) 2016- Facebook, Inc (Adam Paszke) -Copyright (c) 2014- Facebook, Inc (Soumith Chintala) -Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) -Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) -Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) -Copyright (c) 2011-2013 NYU (Clement Farabet) -Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, -Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute -(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, -Samy Bengio, Johnny Mariethoz) - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories -America and IDIAP Research Institute nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. -*/ - -#ifndef SIMDJSON_INTERNAL_INSTRUCTION_SET_H -#define SIMDJSON_INTERNAL_INSTRUCTION_SET_H - -namespace simdjson { -namespace internal { - -enum instruction_set { - DEFAULT = 0x0, - NEON = 0x1, - AVX2 = 0x4, - SSE42 = 0x8, - PCLMULQDQ = 0x10, - BMI1 = 0x20, - BMI2 = 0x40, - ALTIVEC = 0x80, - AVX512F = 0x100, - AVX512DQ = 0x200, - AVX512IFMA = 0x400, - AVX512PF = 0x800, - AVX512ER = 0x1000, - AVX512CD = 0x2000, - AVX512BW = 0x4000, - AVX512VL = 0x8000, - AVX512VBMI2 = 0x10000, - LSX = 0x20000, - LASX = 0x40000, - //RVV = 0x80000, - RVV_VLS = 0x100000, - SVE = 0x200000, - SVE2 = 0x400000, -}; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_INSTRUCTION_SET_H -/* end file simdjson/internal/instruction_set.h */ -/* skipped duplicate #include "simdjson/internal/dom_parser_implementation.h" */ -/* including simdjson/internal/jsoncharutils_tables.h: #include "simdjson/internal/jsoncharutils_tables.h" */ -/* begin file simdjson/internal/jsoncharutils_tables.h */ -#ifndef SIMDJSON_INTERNAL_JSONCHARUTILS_TABLES_H -#define SIMDJSON_INTERNAL_JSONCHARUTILS_TABLES_H - -/* skipped duplicate #include "simdjson/base.h" */ - -#ifdef JSON_TEST_STRINGS -void found_string(const uint8_t *buf, const uint8_t *parsed_begin, - const uint8_t *parsed_end); -void found_bad_string(const uint8_t *buf); -#endif - -namespace simdjson { -namespace internal { -// structural chars here are -// they are { 0x7b } 0x7d : 0x3a [ 0x5b ] 0x5d , 0x2c (and NULL) -// we are also interested in the four whitespace characters -// space 0x20, linefeed 0x0a, horizontal tab 0x09 and carriage return 0x0d - -extern SIMDJSON_DLLIMPORTEXPORT const bool structural_or_whitespace_negated[256]; -extern SIMDJSON_DLLIMPORTEXPORT const bool structural_or_whitespace[256]; -extern SIMDJSON_DLLIMPORTEXPORT const uint32_t digit_to_val32[886]; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_JSONCHARUTILS_TABLES_H -/* end file simdjson/internal/jsoncharutils_tables.h */ -/* including simdjson/internal/numberparsing_tables.h: #include "simdjson/internal/numberparsing_tables.h" */ -/* begin file simdjson/internal/numberparsing_tables.h */ -#ifndef SIMDJSON_INTERNAL_NUMBERPARSING_TABLES_H -#define SIMDJSON_INTERNAL_NUMBERPARSING_TABLES_H - -/* skipped duplicate #include "simdjson/base.h" */ - -namespace simdjson { -namespace internal { -/** - * The smallest non-zero float (binary64) is 2^-1074. - * We take as input numbers of the form w x 10^q where w < 2^64. - * We have that w * 10^-343 < 2^(64-344) 5^-343 < 2^-1076. - * However, we have that - * (2^64-1) * 10^-342 = (2^64-1) * 2^-342 * 5^-342 > 2^-1074. - * Thus it is possible for a number of the form w * 10^-342 where - * w is a 64-bit value to be a non-zero floating-point number. - ********* - * Any number of form w * 10^309 where w>= 1 is going to be - * infinite in binary64 so we never need to worry about powers - * of 5 greater than 308. - */ -constexpr int smallest_power = -342; -constexpr int largest_power = 308; - -/** - * Represents a 128-bit value. - * low: least significant 64 bits. - * high: most significant 64 bits. - */ -struct value128 { - uint64_t low; - uint64_t high; -}; - - -// Precomputed powers of ten from 10^0 to 10^22. These -// can be represented exactly using the double type. -extern SIMDJSON_DLLIMPORTEXPORT const double power_of_ten[]; - - -/** - * When mapping numbers from decimal to binary, - * we go from w * 10^q to m * 2^p but we have - * 10^q = 5^q * 2^q, so effectively - * we are trying to match - * w * 2^q * 5^q to m * 2^p. Thus the powers of two - * are not a concern since they can be represented - * exactly using the binary notation, only the powers of five - * affect the binary significand. - */ - - -// The truncated powers of five from 5^-342 all the way to 5^308 -// The mantissa is truncated to 128 bits, and -// never rounded up. Uses about 10KB. -// We use the template trick to allow inclusion in multiple translation units. -#if SIMDJSON_STATIC_REFLECTION -template struct powers_template { -constexpr static uint64_t power_of_five_128[651*2]= { - 0xeef453d6923bd65a,0x113faa2906a13b3f, - 0x9558b4661b6565f8,0x4ac7ca59a424c507, - 0xbaaee17fa23ebf76,0x5d79bcf00d2df649, - 0xe95a99df8ace6f53,0xf4d82c2c107973dc, - 0x91d8a02bb6c10594,0x79071b9b8a4be869, - 0xb64ec836a47146f9,0x9748e2826cdee284, - 0xe3e27a444d8d98b7,0xfd1b1b2308169b25, - 0x8e6d8c6ab0787f72,0xfe30f0f5e50e20f7, - 0xb208ef855c969f4f,0xbdbd2d335e51a935, - 0xde8b2b66b3bc4723,0xad2c788035e61382, - 0x8b16fb203055ac76,0x4c3bcb5021afcc31, - 0xaddcb9e83c6b1793,0xdf4abe242a1bbf3d, - 0xd953e8624b85dd78,0xd71d6dad34a2af0d, - 0x87d4713d6f33aa6b,0x8672648c40e5ad68, - 0xa9c98d8ccb009506,0x680efdaf511f18c2, - 0xd43bf0effdc0ba48,0x212bd1b2566def2, - 0x84a57695fe98746d,0x14bb630f7604b57, - 0xa5ced43b7e3e9188,0x419ea3bd35385e2d, - 0xcf42894a5dce35ea,0x52064cac828675b9, - 0x818995ce7aa0e1b2,0x7343efebd1940993, - 0xa1ebfb4219491a1f,0x1014ebe6c5f90bf8, - 0xca66fa129f9b60a6,0xd41a26e077774ef6, - 0xfd00b897478238d0,0x8920b098955522b4, - 0x9e20735e8cb16382,0x55b46e5f5d5535b0, - 0xc5a890362fddbc62,0xeb2189f734aa831d, - 0xf712b443bbd52b7b,0xa5e9ec7501d523e4, - 0x9a6bb0aa55653b2d,0x47b233c92125366e, - 0xc1069cd4eabe89f8,0x999ec0bb696e840a, - 0xf148440a256e2c76,0xc00670ea43ca250d, - 0x96cd2a865764dbca,0x380406926a5e5728, - 0xbc807527ed3e12bc,0xc605083704f5ecf2, - 0xeba09271e88d976b,0xf7864a44c633682e, - 0x93445b8731587ea3,0x7ab3ee6afbe0211d, - 0xb8157268fdae9e4c,0x5960ea05bad82964, - 0xe61acf033d1a45df,0x6fb92487298e33bd, - 0x8fd0c16206306bab,0xa5d3b6d479f8e056, - 0xb3c4f1ba87bc8696,0x8f48a4899877186c, - 0xe0b62e2929aba83c,0x331acdabfe94de87, - 0x8c71dcd9ba0b4925,0x9ff0c08b7f1d0b14, - 0xaf8e5410288e1b6f,0x7ecf0ae5ee44dd9, - 0xdb71e91432b1a24a,0xc9e82cd9f69d6150, - 0x892731ac9faf056e,0xbe311c083a225cd2, - 0xab70fe17c79ac6ca,0x6dbd630a48aaf406, - 0xd64d3d9db981787d,0x92cbbccdad5b108, - 0x85f0468293f0eb4e,0x25bbf56008c58ea5, - 0xa76c582338ed2621,0xaf2af2b80af6f24e, - 0xd1476e2c07286faa,0x1af5af660db4aee1, - 0x82cca4db847945ca,0x50d98d9fc890ed4d, - 0xa37fce126597973c,0xe50ff107bab528a0, - 0xcc5fc196fefd7d0c,0x1e53ed49a96272c8, - 0xff77b1fcbebcdc4f,0x25e8e89c13bb0f7a, - 0x9faacf3df73609b1,0x77b191618c54e9ac, - 0xc795830d75038c1d,0xd59df5b9ef6a2417, - 0xf97ae3d0d2446f25,0x4b0573286b44ad1d, - 0x9becce62836ac577,0x4ee367f9430aec32, - 0xc2e801fb244576d5,0x229c41f793cda73f, - 0xf3a20279ed56d48a,0x6b43527578c1110f, - 0x9845418c345644d6,0x830a13896b78aaa9, - 0xbe5691ef416bd60c,0x23cc986bc656d553, - 0xedec366b11c6cb8f,0x2cbfbe86b7ec8aa8, - 0x94b3a202eb1c3f39,0x7bf7d71432f3d6a9, - 0xb9e08a83a5e34f07,0xdaf5ccd93fb0cc53, - 0xe858ad248f5c22c9,0xd1b3400f8f9cff68, - 0x91376c36d99995be,0x23100809b9c21fa1, - 0xb58547448ffffb2d,0xabd40a0c2832a78a, - 0xe2e69915b3fff9f9,0x16c90c8f323f516c, - 0x8dd01fad907ffc3b,0xae3da7d97f6792e3, - 0xb1442798f49ffb4a,0x99cd11cfdf41779c, - 0xdd95317f31c7fa1d,0x40405643d711d583, - 0x8a7d3eef7f1cfc52,0x482835ea666b2572, - 0xad1c8eab5ee43b66,0xda3243650005eecf, - 0xd863b256369d4a40,0x90bed43e40076a82, - 0x873e4f75e2224e68,0x5a7744a6e804a291, - 0xa90de3535aaae202,0x711515d0a205cb36, - 0xd3515c2831559a83,0xd5a5b44ca873e03, - 0x8412d9991ed58091,0xe858790afe9486c2, - 0xa5178fff668ae0b6,0x626e974dbe39a872, - 0xce5d73ff402d98e3,0xfb0a3d212dc8128f, - 0x80fa687f881c7f8e,0x7ce66634bc9d0b99, - 0xa139029f6a239f72,0x1c1fffc1ebc44e80, - 0xc987434744ac874e,0xa327ffb266b56220, - 0xfbe9141915d7a922,0x4bf1ff9f0062baa8, - 0x9d71ac8fada6c9b5,0x6f773fc3603db4a9, - 0xc4ce17b399107c22,0xcb550fb4384d21d3, - 0xf6019da07f549b2b,0x7e2a53a146606a48, - 0x99c102844f94e0fb,0x2eda7444cbfc426d, - 0xc0314325637a1939,0xfa911155fefb5308, - 0xf03d93eebc589f88,0x793555ab7eba27ca, - 0x96267c7535b763b5,0x4bc1558b2f3458de, - 0xbbb01b9283253ca2,0x9eb1aaedfb016f16, - 0xea9c227723ee8bcb,0x465e15a979c1cadc, - 0x92a1958a7675175f,0xbfacd89ec191ec9, - 0xb749faed14125d36,0xcef980ec671f667b, - 0xe51c79a85916f484,0x82b7e12780e7401a, - 0x8f31cc0937ae58d2,0xd1b2ecb8b0908810, - 0xb2fe3f0b8599ef07,0x861fa7e6dcb4aa15, - 0xdfbdcece67006ac9,0x67a791e093e1d49a, - 0x8bd6a141006042bd,0xe0c8bb2c5c6d24e0, - 0xaecc49914078536d,0x58fae9f773886e18, - 0xda7f5bf590966848,0xaf39a475506a899e, - 0x888f99797a5e012d,0x6d8406c952429603, - 0xaab37fd7d8f58178,0xc8e5087ba6d33b83, - 0xd5605fcdcf32e1d6,0xfb1e4a9a90880a64, - 0x855c3be0a17fcd26,0x5cf2eea09a55067f, - 0xa6b34ad8c9dfc06f,0xf42faa48c0ea481e, - 0xd0601d8efc57b08b,0xf13b94daf124da26, - 0x823c12795db6ce57,0x76c53d08d6b70858, - 0xa2cb1717b52481ed,0x54768c4b0c64ca6e, - 0xcb7ddcdda26da268,0xa9942f5dcf7dfd09, - 0xfe5d54150b090b02,0xd3f93b35435d7c4c, - 0x9efa548d26e5a6e1,0xc47bc5014a1a6daf, - 0xc6b8e9b0709f109a,0x359ab6419ca1091b, - 0xf867241c8cc6d4c0,0xc30163d203c94b62, - 0x9b407691d7fc44f8,0x79e0de63425dcf1d, - 0xc21094364dfb5636,0x985915fc12f542e4, - 0xf294b943e17a2bc4,0x3e6f5b7b17b2939d, - 0x979cf3ca6cec5b5a,0xa705992ceecf9c42, - 0xbd8430bd08277231,0x50c6ff782a838353, - 0xece53cec4a314ebd,0xa4f8bf5635246428, - 0x940f4613ae5ed136,0x871b7795e136be99, - 0xb913179899f68584,0x28e2557b59846e3f, - 0xe757dd7ec07426e5,0x331aeada2fe589cf, - 0x9096ea6f3848984f,0x3ff0d2c85def7621, - 0xb4bca50b065abe63,0xfed077a756b53a9, - 0xe1ebce4dc7f16dfb,0xd3e8495912c62894, - 0x8d3360f09cf6e4bd,0x64712dd7abbbd95c, - 0xb080392cc4349dec,0xbd8d794d96aacfb3, - 0xdca04777f541c567,0xecf0d7a0fc5583a0, - 0x89e42caaf9491b60,0xf41686c49db57244, - 0xac5d37d5b79b6239,0x311c2875c522ced5, - 0xd77485cb25823ac7,0x7d633293366b828b, - 0x86a8d39ef77164bc,0xae5dff9c02033197, - 0xa8530886b54dbdeb,0xd9f57f830283fdfc, - 0xd267caa862a12d66,0xd072df63c324fd7b, - 0x8380dea93da4bc60,0x4247cb9e59f71e6d, - 0xa46116538d0deb78,0x52d9be85f074e608, - 0xcd795be870516656,0x67902e276c921f8b, - 0x806bd9714632dff6,0xba1cd8a3db53b6, - 0xa086cfcd97bf97f3,0x80e8a40eccd228a4, - 0xc8a883c0fdaf7df0,0x6122cd128006b2cd, - 0xfad2a4b13d1b5d6c,0x796b805720085f81, - 0x9cc3a6eec6311a63,0xcbe3303674053bb0, - 0xc3f490aa77bd60fc,0xbedbfc4411068a9c, - 0xf4f1b4d515acb93b,0xee92fb5515482d44, - 0x991711052d8bf3c5,0x751bdd152d4d1c4a, - 0xbf5cd54678eef0b6,0xd262d45a78a0635d, - 0xef340a98172aace4,0x86fb897116c87c34, - 0x9580869f0e7aac0e,0xd45d35e6ae3d4da0, - 0xbae0a846d2195712,0x8974836059cca109, - 0xe998d258869facd7,0x2bd1a438703fc94b, - 0x91ff83775423cc06,0x7b6306a34627ddcf, - 0xb67f6455292cbf08,0x1a3bc84c17b1d542, - 0xe41f3d6a7377eeca,0x20caba5f1d9e4a93, - 0x8e938662882af53e,0x547eb47b7282ee9c, - 0xb23867fb2a35b28d,0xe99e619a4f23aa43, - 0xdec681f9f4c31f31,0x6405fa00e2ec94d4, - 0x8b3c113c38f9f37e,0xde83bc408dd3dd04, - 0xae0b158b4738705e,0x9624ab50b148d445, - 0xd98ddaee19068c76,0x3badd624dd9b0957, - 0x87f8a8d4cfa417c9,0xe54ca5d70a80e5d6, - 0xa9f6d30a038d1dbc,0x5e9fcf4ccd211f4c, - 0xd47487cc8470652b,0x7647c3200069671f, - 0x84c8d4dfd2c63f3b,0x29ecd9f40041e073, - 0xa5fb0a17c777cf09,0xf468107100525890, - 0xcf79cc9db955c2cc,0x7182148d4066eeb4, - 0x81ac1fe293d599bf,0xc6f14cd848405530, - 0xa21727db38cb002f,0xb8ada00e5a506a7c, - 0xca9cf1d206fdc03b,0xa6d90811f0e4851c, - 0xfd442e4688bd304a,0x908f4a166d1da663, - 0x9e4a9cec15763e2e,0x9a598e4e043287fe, - 0xc5dd44271ad3cdba,0x40eff1e1853f29fd, - 0xf7549530e188c128,0xd12bee59e68ef47c, - 0x9a94dd3e8cf578b9,0x82bb74f8301958ce, - 0xc13a148e3032d6e7,0xe36a52363c1faf01, - 0xf18899b1bc3f8ca1,0xdc44e6c3cb279ac1, - 0x96f5600f15a7b7e5,0x29ab103a5ef8c0b9, - 0xbcb2b812db11a5de,0x7415d448f6b6f0e7, - 0xebdf661791d60f56,0x111b495b3464ad21, - 0x936b9fcebb25c995,0xcab10dd900beec34, - 0xb84687c269ef3bfb,0x3d5d514f40eea742, - 0xe65829b3046b0afa,0xcb4a5a3112a5112, - 0x8ff71a0fe2c2e6dc,0x47f0e785eaba72ab, - 0xb3f4e093db73a093,0x59ed216765690f56, - 0xe0f218b8d25088b8,0x306869c13ec3532c, - 0x8c974f7383725573,0x1e414218c73a13fb, - 0xafbd2350644eeacf,0xe5d1929ef90898fa, - 0xdbac6c247d62a583,0xdf45f746b74abf39, - 0x894bc396ce5da772,0x6b8bba8c328eb783, - 0xab9eb47c81f5114f,0x66ea92f3f326564, - 0xd686619ba27255a2,0xc80a537b0efefebd, - 0x8613fd0145877585,0xbd06742ce95f5f36, - 0xa798fc4196e952e7,0x2c48113823b73704, - 0xd17f3b51fca3a7a0,0xf75a15862ca504c5, - 0x82ef85133de648c4,0x9a984d73dbe722fb, - 0xa3ab66580d5fdaf5,0xc13e60d0d2e0ebba, - 0xcc963fee10b7d1b3,0x318df905079926a8, - 0xffbbcfe994e5c61f,0xfdf17746497f7052, - 0x9fd561f1fd0f9bd3,0xfeb6ea8bedefa633, - 0xc7caba6e7c5382c8,0xfe64a52ee96b8fc0, - 0xf9bd690a1b68637b,0x3dfdce7aa3c673b0, - 0x9c1661a651213e2d,0x6bea10ca65c084e, - 0xc31bfa0fe5698db8,0x486e494fcff30a62, - 0xf3e2f893dec3f126,0x5a89dba3c3efccfa, - 0x986ddb5c6b3a76b7,0xf89629465a75e01c, - 0xbe89523386091465,0xf6bbb397f1135823, - 0xee2ba6c0678b597f,0x746aa07ded582e2c, - 0x94db483840b717ef,0xa8c2a44eb4571cdc, - 0xba121a4650e4ddeb,0x92f34d62616ce413, - 0xe896a0d7e51e1566,0x77b020baf9c81d17, - 0x915e2486ef32cd60,0xace1474dc1d122e, - 0xb5b5ada8aaff80b8,0xd819992132456ba, - 0xe3231912d5bf60e6,0x10e1fff697ed6c69, - 0x8df5efabc5979c8f,0xca8d3ffa1ef463c1, - 0xb1736b96b6fd83b3,0xbd308ff8a6b17cb2, - 0xddd0467c64bce4a0,0xac7cb3f6d05ddbde, - 0x8aa22c0dbef60ee4,0x6bcdf07a423aa96b, - 0xad4ab7112eb3929d,0x86c16c98d2c953c6, - 0xd89d64d57a607744,0xe871c7bf077ba8b7, - 0x87625f056c7c4a8b,0x11471cd764ad4972, - 0xa93af6c6c79b5d2d,0xd598e40d3dd89bcf, - 0xd389b47879823479,0x4aff1d108d4ec2c3, - 0x843610cb4bf160cb,0xcedf722a585139ba, - 0xa54394fe1eedb8fe,0xc2974eb4ee658828, - 0xce947a3da6a9273e,0x733d226229feea32, - 0x811ccc668829b887,0x806357d5a3f525f, - 0xa163ff802a3426a8,0xca07c2dcb0cf26f7, - 0xc9bcff6034c13052,0xfc89b393dd02f0b5, - 0xfc2c3f3841f17c67,0xbbac2078d443ace2, - 0x9d9ba7832936edc0,0xd54b944b84aa4c0d, - 0xc5029163f384a931,0xa9e795e65d4df11, - 0xf64335bcf065d37d,0x4d4617b5ff4a16d5, - 0x99ea0196163fa42e,0x504bced1bf8e4e45, - 0xc06481fb9bcf8d39,0xe45ec2862f71e1d6, - 0xf07da27a82c37088,0x5d767327bb4e5a4c, - 0x964e858c91ba2655,0x3a6a07f8d510f86f, - 0xbbe226efb628afea,0x890489f70a55368b, - 0xeadab0aba3b2dbe5,0x2b45ac74ccea842e, - 0x92c8ae6b464fc96f,0x3b0b8bc90012929d, - 0xb77ada0617e3bbcb,0x9ce6ebb40173744, - 0xe55990879ddcaabd,0xcc420a6a101d0515, - 0x8f57fa54c2a9eab6,0x9fa946824a12232d, - 0xb32df8e9f3546564,0x47939822dc96abf9, - 0xdff9772470297ebd,0x59787e2b93bc56f7, - 0x8bfbea76c619ef36,0x57eb4edb3c55b65a, - 0xaefae51477a06b03,0xede622920b6b23f1, - 0xdab99e59958885c4,0xe95fab368e45eced, - 0x88b402f7fd75539b,0x11dbcb0218ebb414, - 0xaae103b5fcd2a881,0xd652bdc29f26a119, - 0xd59944a37c0752a2,0x4be76d3346f0495f, - 0x857fcae62d8493a5,0x6f70a4400c562ddb, - 0xa6dfbd9fb8e5b88e,0xcb4ccd500f6bb952, - 0xd097ad07a71f26b2,0x7e2000a41346a7a7, - 0x825ecc24c873782f,0x8ed400668c0c28c8, - 0xa2f67f2dfa90563b,0x728900802f0f32fa, - 0xcbb41ef979346bca,0x4f2b40a03ad2ffb9, - 0xfea126b7d78186bc,0xe2f610c84987bfa8, - 0x9f24b832e6b0f436,0xdd9ca7d2df4d7c9, - 0xc6ede63fa05d3143,0x91503d1c79720dbb, - 0xf8a95fcf88747d94,0x75a44c6397ce912a, - 0x9b69dbe1b548ce7c,0xc986afbe3ee11aba, - 0xc24452da229b021b,0xfbe85badce996168, - 0xf2d56790ab41c2a2,0xfae27299423fb9c3, - 0x97c560ba6b0919a5,0xdccd879fc967d41a, - 0xbdb6b8e905cb600f,0x5400e987bbc1c920, - 0xed246723473e3813,0x290123e9aab23b68, - 0x9436c0760c86e30b,0xf9a0b6720aaf6521, - 0xb94470938fa89bce,0xf808e40e8d5b3e69, - 0xe7958cb87392c2c2,0xb60b1d1230b20e04, - 0x90bd77f3483bb9b9,0xb1c6f22b5e6f48c2, - 0xb4ecd5f01a4aa828,0x1e38aeb6360b1af3, - 0xe2280b6c20dd5232,0x25c6da63c38de1b0, - 0x8d590723948a535f,0x579c487e5a38ad0e, - 0xb0af48ec79ace837,0x2d835a9df0c6d851, - 0xdcdb1b2798182244,0xf8e431456cf88e65, - 0x8a08f0f8bf0f156b,0x1b8e9ecb641b58ff, - 0xac8b2d36eed2dac5,0xe272467e3d222f3f, - 0xd7adf884aa879177,0x5b0ed81dcc6abb0f, - 0x86ccbb52ea94baea,0x98e947129fc2b4e9, - 0xa87fea27a539e9a5,0x3f2398d747b36224, - 0xd29fe4b18e88640e,0x8eec7f0d19a03aad, - 0x83a3eeeef9153e89,0x1953cf68300424ac, - 0xa48ceaaab75a8e2b,0x5fa8c3423c052dd7, - 0xcdb02555653131b6,0x3792f412cb06794d, - 0x808e17555f3ebf11,0xe2bbd88bbee40bd0, - 0xa0b19d2ab70e6ed6,0x5b6aceaeae9d0ec4, - 0xc8de047564d20a8b,0xf245825a5a445275, - 0xfb158592be068d2e,0xeed6e2f0f0d56712, - 0x9ced737bb6c4183d,0x55464dd69685606b, - 0xc428d05aa4751e4c,0xaa97e14c3c26b886, - 0xf53304714d9265df,0xd53dd99f4b3066a8, - 0x993fe2c6d07b7fab,0xe546a8038efe4029, - 0xbf8fdb78849a5f96,0xde98520472bdd033, - 0xef73d256a5c0f77c,0x963e66858f6d4440, - 0x95a8637627989aad,0xdde7001379a44aa8, - 0xbb127c53b17ec159,0x5560c018580d5d52, - 0xe9d71b689dde71af,0xaab8f01e6e10b4a6, - 0x9226712162ab070d,0xcab3961304ca70e8, - 0xb6b00d69bb55c8d1,0x3d607b97c5fd0d22, - 0xe45c10c42a2b3b05,0x8cb89a7db77c506a, - 0x8eb98a7a9a5b04e3,0x77f3608e92adb242, - 0xb267ed1940f1c61c,0x55f038b237591ed3, - 0xdf01e85f912e37a3,0x6b6c46dec52f6688, - 0x8b61313bbabce2c6,0x2323ac4b3b3da015, - 0xae397d8aa96c1b77,0xabec975e0a0d081a, - 0xd9c7dced53c72255,0x96e7bd358c904a21, - 0x881cea14545c7575,0x7e50d64177da2e54, - 0xaa242499697392d2,0xdde50bd1d5d0b9e9, - 0xd4ad2dbfc3d07787,0x955e4ec64b44e864, - 0x84ec3c97da624ab4,0xbd5af13bef0b113e, - 0xa6274bbdd0fadd61,0xecb1ad8aeacdd58e, - 0xcfb11ead453994ba,0x67de18eda5814af2, - 0x81ceb32c4b43fcf4,0x80eacf948770ced7, - 0xa2425ff75e14fc31,0xa1258379a94d028d, - 0xcad2f7f5359a3b3e,0x96ee45813a04330, - 0xfd87b5f28300ca0d,0x8bca9d6e188853fc, - 0x9e74d1b791e07e48,0x775ea264cf55347e, - 0xc612062576589dda,0x95364afe032a81a0, - 0xf79687aed3eec551,0x3a83ddbd83f52210, - 0x9abe14cd44753b52,0xc4926a9672793580, - 0xc16d9a0095928a27,0x75b7053c0f178400, - 0xf1c90080baf72cb1,0x5324c68b12dd6800, - 0x971da05074da7bee,0xd3f6fc16ebca8000, - 0xbce5086492111aea,0x88f4bb1ca6bd0000, - 0xec1e4a7db69561a5,0x2b31e9e3d0700000, - 0x9392ee8e921d5d07,0x3aff322e62600000, - 0xb877aa3236a4b449,0x9befeb9fad487c3, - 0xe69594bec44de15b,0x4c2ebe687989a9b4, - 0x901d7cf73ab0acd9,0xf9d37014bf60a11, - 0xb424dc35095cd80f,0x538484c19ef38c95, - 0xe12e13424bb40e13,0x2865a5f206b06fba, - 0x8cbccc096f5088cb,0xf93f87b7442e45d4, - 0xafebff0bcb24aafe,0xf78f69a51539d749, - 0xdbe6fecebdedd5be,0xb573440e5a884d1c, - 0x89705f4136b4a597,0x31680a88f8953031, - 0xabcc77118461cefc,0xfdc20d2b36ba7c3e, - 0xd6bf94d5e57a42bc,0x3d32907604691b4d, - 0x8637bd05af6c69b5,0xa63f9a49c2c1b110, - 0xa7c5ac471b478423,0xfcf80dc33721d54, - 0xd1b71758e219652b,0xd3c36113404ea4a9, - 0x83126e978d4fdf3b,0x645a1cac083126ea, - 0xa3d70a3d70a3d70a,0x3d70a3d70a3d70a4, - 0xcccccccccccccccc,0xcccccccccccccccd, - 0x8000000000000000,0x0, - 0xa000000000000000,0x0, - 0xc800000000000000,0x0, - 0xfa00000000000000,0x0, - 0x9c40000000000000,0x0, - 0xc350000000000000,0x0, - 0xf424000000000000,0x0, - 0x9896800000000000,0x0, - 0xbebc200000000000,0x0, - 0xee6b280000000000,0x0, - 0x9502f90000000000,0x0, - 0xba43b74000000000,0x0, - 0xe8d4a51000000000,0x0, - 0x9184e72a00000000,0x0, - 0xb5e620f480000000,0x0, - 0xe35fa931a0000000,0x0, - 0x8e1bc9bf04000000,0x0, - 0xb1a2bc2ec5000000,0x0, - 0xde0b6b3a76400000,0x0, - 0x8ac7230489e80000,0x0, - 0xad78ebc5ac620000,0x0, - 0xd8d726b7177a8000,0x0, - 0x878678326eac9000,0x0, - 0xa968163f0a57b400,0x0, - 0xd3c21bcecceda100,0x0, - 0x84595161401484a0,0x0, - 0xa56fa5b99019a5c8,0x0, - 0xcecb8f27f4200f3a,0x0, - 0x813f3978f8940984,0x4000000000000000, - 0xa18f07d736b90be5,0x5000000000000000, - 0xc9f2c9cd04674ede,0xa400000000000000, - 0xfc6f7c4045812296,0x4d00000000000000, - 0x9dc5ada82b70b59d,0xf020000000000000, - 0xc5371912364ce305,0x6c28000000000000, - 0xf684df56c3e01bc6,0xc732000000000000, - 0x9a130b963a6c115c,0x3c7f400000000000, - 0xc097ce7bc90715b3,0x4b9f100000000000, - 0xf0bdc21abb48db20,0x1e86d40000000000, - 0x96769950b50d88f4,0x1314448000000000, - 0xbc143fa4e250eb31,0x17d955a000000000, - 0xeb194f8e1ae525fd,0x5dcfab0800000000, - 0x92efd1b8d0cf37be,0x5aa1cae500000000, - 0xb7abc627050305ad,0xf14a3d9e40000000, - 0xe596b7b0c643c719,0x6d9ccd05d0000000, - 0x8f7e32ce7bea5c6f,0xe4820023a2000000, - 0xb35dbf821ae4f38b,0xdda2802c8a800000, - 0xe0352f62a19e306e,0xd50b2037ad200000, - 0x8c213d9da502de45,0x4526f422cc340000, - 0xaf298d050e4395d6,0x9670b12b7f410000, - 0xdaf3f04651d47b4c,0x3c0cdd765f114000, - 0x88d8762bf324cd0f,0xa5880a69fb6ac800, - 0xab0e93b6efee0053,0x8eea0d047a457a00, - 0xd5d238a4abe98068,0x72a4904598d6d880, - 0x85a36366eb71f041,0x47a6da2b7f864750, - 0xa70c3c40a64e6c51,0x999090b65f67d924, - 0xd0cf4b50cfe20765,0xfff4b4e3f741cf6d, - 0x82818f1281ed449f,0xbff8f10e7a8921a4, - 0xa321f2d7226895c7,0xaff72d52192b6a0d, - 0xcbea6f8ceb02bb39,0x9bf4f8a69f764490, - 0xfee50b7025c36a08,0x2f236d04753d5b4, - 0x9f4f2726179a2245,0x1d762422c946590, - 0xc722f0ef9d80aad6,0x424d3ad2b7b97ef5, - 0xf8ebad2b84e0d58b,0xd2e0898765a7deb2, - 0x9b934c3b330c8577,0x63cc55f49f88eb2f, - 0xc2781f49ffcfa6d5,0x3cbf6b71c76b25fb, - 0xf316271c7fc3908a,0x8bef464e3945ef7a, - 0x97edd871cfda3a56,0x97758bf0e3cbb5ac, - 0xbde94e8e43d0c8ec,0x3d52eeed1cbea317, - 0xed63a231d4c4fb27,0x4ca7aaa863ee4bdd, - 0x945e455f24fb1cf8,0x8fe8caa93e74ef6a, - 0xb975d6b6ee39e436,0xb3e2fd538e122b44, - 0xe7d34c64a9c85d44,0x60dbbca87196b616, - 0x90e40fbeea1d3a4a,0xbc8955e946fe31cd, - 0xb51d13aea4a488dd,0x6babab6398bdbe41, - 0xe264589a4dcdab14,0xc696963c7eed2dd1, - 0x8d7eb76070a08aec,0xfc1e1de5cf543ca2, - 0xb0de65388cc8ada8,0x3b25a55f43294bcb, - 0xdd15fe86affad912,0x49ef0eb713f39ebe, - 0x8a2dbf142dfcc7ab,0x6e3569326c784337, - 0xacb92ed9397bf996,0x49c2c37f07965404, - 0xd7e77a8f87daf7fb,0xdc33745ec97be906, - 0x86f0ac99b4e8dafd,0x69a028bb3ded71a3, - 0xa8acd7c0222311bc,0xc40832ea0d68ce0c, - 0xd2d80db02aabd62b,0xf50a3fa490c30190, - 0x83c7088e1aab65db,0x792667c6da79e0fa, - 0xa4b8cab1a1563f52,0x577001b891185938, - 0xcde6fd5e09abcf26,0xed4c0226b55e6f86, - 0x80b05e5ac60b6178,0x544f8158315b05b4, - 0xa0dc75f1778e39d6,0x696361ae3db1c721, - 0xc913936dd571c84c,0x3bc3a19cd1e38e9, - 0xfb5878494ace3a5f,0x4ab48a04065c723, - 0x9d174b2dcec0e47b,0x62eb0d64283f9c76, - 0xc45d1df942711d9a,0x3ba5d0bd324f8394, - 0xf5746577930d6500,0xca8f44ec7ee36479, - 0x9968bf6abbe85f20,0x7e998b13cf4e1ecb, - 0xbfc2ef456ae276e8,0x9e3fedd8c321a67e, - 0xefb3ab16c59b14a2,0xc5cfe94ef3ea101e, - 0x95d04aee3b80ece5,0xbba1f1d158724a12, - 0xbb445da9ca61281f,0x2a8a6e45ae8edc97, - 0xea1575143cf97226,0xf52d09d71a3293bd, - 0x924d692ca61be758,0x593c2626705f9c56, - 0xb6e0c377cfa2e12e,0x6f8b2fb00c77836c, - 0xe498f455c38b997a,0xb6dfb9c0f956447, - 0x8edf98b59a373fec,0x4724bd4189bd5eac, - 0xb2977ee300c50fe7,0x58edec91ec2cb657, - 0xdf3d5e9bc0f653e1,0x2f2967b66737e3ed, - 0x8b865b215899f46c,0xbd79e0d20082ee74, - 0xae67f1e9aec07187,0xecd8590680a3aa11, - 0xda01ee641a708de9,0xe80e6f4820cc9495, - 0x884134fe908658b2,0x3109058d147fdcdd, - 0xaa51823e34a7eede,0xbd4b46f0599fd415, - 0xd4e5e2cdc1d1ea96,0x6c9e18ac7007c91a, - 0x850fadc09923329e,0x3e2cf6bc604ddb0, - 0xa6539930bf6bff45,0x84db8346b786151c, - 0xcfe87f7cef46ff16,0xe612641865679a63, - 0x81f14fae158c5f6e,0x4fcb7e8f3f60c07e, - 0xa26da3999aef7749,0xe3be5e330f38f09d, - 0xcb090c8001ab551c,0x5cadf5bfd3072cc5, - 0xfdcb4fa002162a63,0x73d9732fc7c8f7f6, - 0x9e9f11c4014dda7e,0x2867e7fddcdd9afa, - 0xc646d63501a1511d,0xb281e1fd541501b8, - 0xf7d88bc24209a565,0x1f225a7ca91a4226, - 0x9ae757596946075f,0x3375788de9b06958, - 0xc1a12d2fc3978937,0x52d6b1641c83ae, - 0xf209787bb47d6b84,0xc0678c5dbd23a49a, - 0x9745eb4d50ce6332,0xf840b7ba963646e0, - 0xbd176620a501fbff,0xb650e5a93bc3d898, - 0xec5d3fa8ce427aff,0xa3e51f138ab4cebe, - 0x93ba47c980e98cdf,0xc66f336c36b10137, - 0xb8a8d9bbe123f017,0xb80b0047445d4184, - 0xe6d3102ad96cec1d,0xa60dc059157491e5, - 0x9043ea1ac7e41392,0x87c89837ad68db2f, - 0xb454e4a179dd1877,0x29babe4598c311fb, - 0xe16a1dc9d8545e94,0xf4296dd6fef3d67a, - 0x8ce2529e2734bb1d,0x1899e4a65f58660c, - 0xb01ae745b101e9e4,0x5ec05dcff72e7f8f, - 0xdc21a1171d42645d,0x76707543f4fa1f73, - 0x899504ae72497eba,0x6a06494a791c53a8, - 0xabfa45da0edbde69,0x487db9d17636892, - 0xd6f8d7509292d603,0x45a9d2845d3c42b6, - 0x865b86925b9bc5c2,0xb8a2392ba45a9b2, - 0xa7f26836f282b732,0x8e6cac7768d7141e, - 0xd1ef0244af2364ff,0x3207d795430cd926, - 0x8335616aed761f1f,0x7f44e6bd49e807b8, - 0xa402b9c5a8d3a6e7,0x5f16206c9c6209a6, - 0xcd036837130890a1,0x36dba887c37a8c0f, - 0x802221226be55a64,0xc2494954da2c9789, - 0xa02aa96b06deb0fd,0xf2db9baa10b7bd6c, - 0xc83553c5c8965d3d,0x6f92829494e5acc7, - 0xfa42a8b73abbf48c,0xcb772339ba1f17f9, - 0x9c69a97284b578d7,0xff2a760414536efb, - 0xc38413cf25e2d70d,0xfef5138519684aba, - 0xf46518c2ef5b8cd1,0x7eb258665fc25d69, - 0x98bf2f79d5993802,0xef2f773ffbd97a61, - 0xbeeefb584aff8603,0xaafb550ffacfd8fa, - 0xeeaaba2e5dbf6784,0x95ba2a53f983cf38, - 0x952ab45cfa97a0b2,0xdd945a747bf26183, - 0xba756174393d88df,0x94f971119aeef9e4, - 0xe912b9d1478ceb17,0x7a37cd5601aab85d, - 0x91abb422ccb812ee,0xac62e055c10ab33a, - 0xb616a12b7fe617aa,0x577b986b314d6009, - 0xe39c49765fdf9d94,0xed5a7e85fda0b80b, - 0x8e41ade9fbebc27d,0x14588f13be847307, - 0xb1d219647ae6b31c,0x596eb2d8ae258fc8, - 0xde469fbd99a05fe3,0x6fca5f8ed9aef3bb, - 0x8aec23d680043bee,0x25de7bb9480d5854, - 0xada72ccc20054ae9,0xaf561aa79a10ae6a, - 0xd910f7ff28069da4,0x1b2ba1518094da04, - 0x87aa9aff79042286,0x90fb44d2f05d0842, - 0xa99541bf57452b28,0x353a1607ac744a53, - 0xd3fa922f2d1675f2,0x42889b8997915ce8, - 0x847c9b5d7c2e09b7,0x69956135febada11, - 0xa59bc234db398c25,0x43fab9837e699095, - 0xcf02b2c21207ef2e,0x94f967e45e03f4bb, - 0x8161afb94b44f57d,0x1d1be0eebac278f5, - 0xa1ba1ba79e1632dc,0x6462d92a69731732, - 0xca28a291859bbf93,0x7d7b8f7503cfdcfe, - 0xfcb2cb35e702af78,0x5cda735244c3d43e, - 0x9defbf01b061adab,0x3a0888136afa64a7, - 0xc56baec21c7a1916,0x88aaa1845b8fdd0, - 0xf6c69a72a3989f5b,0x8aad549e57273d45, - 0x9a3c2087a63f6399,0x36ac54e2f678864b, - 0xc0cb28a98fcf3c7f,0x84576a1bb416a7dd, - 0xf0fdf2d3f3c30b9f,0x656d44a2a11c51d5, - 0x969eb7c47859e743,0x9f644ae5a4b1b325, - 0xbc4665b596706114,0x873d5d9f0dde1fee, - 0xeb57ff22fc0c7959,0xa90cb506d155a7ea, - 0x9316ff75dd87cbd8,0x9a7f12442d588f2, - 0xb7dcbf5354e9bece,0xc11ed6d538aeb2f, - 0xe5d3ef282a242e81,0x8f1668c8a86da5fa, - 0x8fa475791a569d10,0xf96e017d694487bc, - 0xb38d92d760ec4455,0x37c981dcc395a9ac, - 0xe070f78d3927556a,0x85bbe253f47b1417, - 0x8c469ab843b89562,0x93956d7478ccec8e, - 0xaf58416654a6babb,0x387ac8d1970027b2, - 0xdb2e51bfe9d0696a,0x6997b05fcc0319e, - 0x88fcf317f22241e2,0x441fece3bdf81f03, - 0xab3c2fddeeaad25a,0xd527e81cad7626c3, - 0xd60b3bd56a5586f1,0x8a71e223d8d3b074, - 0x85c7056562757456,0xf6872d5667844e49, - 0xa738c6bebb12d16c,0xb428f8ac016561db, - 0xd106f86e69d785c7,0xe13336d701beba52, - 0x82a45b450226b39c,0xecc0024661173473, - 0xa34d721642b06084,0x27f002d7f95d0190, - 0xcc20ce9bd35c78a5,0x31ec038df7b441f4, - 0xff290242c83396ce,0x7e67047175a15271, - 0x9f79a169bd203e41,0xf0062c6e984d386, - 0xc75809c42c684dd1,0x52c07b78a3e60868, - 0xf92e0c3537826145,0xa7709a56ccdf8a82, - 0x9bbcc7a142b17ccb,0x88a66076400bb691, - 0xc2abf989935ddbfe,0x6acff893d00ea435, - 0xf356f7ebf83552fe,0x583f6b8c4124d43, - 0x98165af37b2153de,0xc3727a337a8b704a, - 0xbe1bf1b059e9a8d6,0x744f18c0592e4c5c, - 0xeda2ee1c7064130c,0x1162def06f79df73, - 0x9485d4d1c63e8be7,0x8addcb5645ac2ba8, - 0xb9a74a0637ce2ee1,0x6d953e2bd7173692, - 0xe8111c87c5c1ba99,0xc8fa8db6ccdd0437, - 0x910ab1d4db9914a0,0x1d9c9892400a22a2, - 0xb54d5e4a127f59c8,0x2503beb6d00cab4b, - 0xe2a0b5dc971f303a,0x2e44ae64840fd61d, - 0x8da471a9de737e24,0x5ceaecfed289e5d2, - 0xb10d8e1456105dad,0x7425a83e872c5f47, - 0xdd50f1996b947518,0xd12f124e28f77719, - 0x8a5296ffe33cc92f,0x82bd6b70d99aaa6f, - 0xace73cbfdc0bfb7b,0x636cc64d1001550b, - 0xd8210befd30efa5a,0x3c47f7e05401aa4e, - 0x8714a775e3e95c78,0x65acfaec34810a71, - 0xa8d9d1535ce3b396,0x7f1839a741a14d0d, - 0xd31045a8341ca07c,0x1ede48111209a050, - 0x83ea2b892091e44d,0x934aed0aab460432, - 0xa4e4b66b68b65d60,0xf81da84d5617853f, - 0xce1de40642e3f4b9,0x36251260ab9d668e, - 0x80d2ae83e9ce78f3,0xc1d72b7c6b426019, - 0xa1075a24e4421730,0xb24cf65b8612f81f, - 0xc94930ae1d529cfc,0xdee033f26797b627, - 0xfb9b7cd9a4a7443c,0x169840ef017da3b1, - 0x9d412e0806e88aa5,0x8e1f289560ee864e, - 0xc491798a08a2ad4e,0xf1a6f2bab92a27e2, - 0xf5b5d7ec8acb58a2,0xae10af696774b1db, - 0x9991a6f3d6bf1765,0xacca6da1e0a8ef29, - 0xbff610b0cc6edd3f,0x17fd090a58d32af3, - 0xeff394dcff8a948e,0xddfc4b4cef07f5b0, - 0x95f83d0a1fb69cd9,0x4abdaf101564f98e, - 0xbb764c4ca7a4440f,0x9d6d1ad41abe37f1, - 0xea53df5fd18d5513,0x84c86189216dc5ed, - 0x92746b9be2f8552c,0x32fd3cf5b4e49bb4, - 0xb7118682dbb66a77,0x3fbc8c33221dc2a1, - 0xe4d5e82392a40515,0xfabaf3feaa5334a, - 0x8f05b1163ba6832d,0x29cb4d87f2a7400e, - 0xb2c71d5bca9023f8,0x743e20e9ef511012, - 0xdf78e4b2bd342cf6,0x914da9246b255416, - 0x8bab8eefb6409c1a,0x1ad089b6c2f7548e, - 0xae9672aba3d0c320,0xa184ac2473b529b1, - 0xda3c0f568cc4f3e8,0xc9e5d72d90a2741e, - 0x8865899617fb1871,0x7e2fa67c7a658892, - 0xaa7eebfb9df9de8d,0xddbb901b98feeab7, - 0xd51ea6fa85785631,0x552a74227f3ea565, - 0x8533285c936b35de,0xd53a88958f87275f, - 0xa67ff273b8460356,0x8a892abaf368f137, - 0xd01fef10a657842c,0x2d2b7569b0432d85, - 0x8213f56a67f6b29b,0x9c3b29620e29fc73, - 0xa298f2c501f45f42,0x8349f3ba91b47b8f, - 0xcb3f2f7642717713,0x241c70a936219a73, - 0xfe0efb53d30dd4d7,0xed238cd383aa0110, - 0x9ec95d1463e8a506,0xf4363804324a40aa, - 0xc67bb4597ce2ce48,0xb143c6053edcd0d5, - 0xf81aa16fdc1b81da,0xdd94b7868e94050a, - 0x9b10a4e5e9913128,0xca7cf2b4191c8326, - 0xc1d4ce1f63f57d72,0xfd1c2f611f63a3f0, - 0xf24a01a73cf2dccf,0xbc633b39673c8cec, - 0x976e41088617ca01,0xd5be0503e085d813, - 0xbd49d14aa79dbc82,0x4b2d8644d8a74e18, - 0xec9c459d51852ba2,0xddf8e7d60ed1219e, - 0x93e1ab8252f33b45,0xcabb90e5c942b503, - 0xb8da1662e7b00a17,0x3d6a751f3b936243, - 0xe7109bfba19c0c9d,0xcc512670a783ad4, - 0x906a617d450187e2,0x27fb2b80668b24c5, - 0xb484f9dc9641e9da,0xb1f9f660802dedf6, - 0xe1a63853bbd26451,0x5e7873f8a0396973, - 0x8d07e33455637eb2,0xdb0b487b6423e1e8, - 0xb049dc016abc5e5f,0x91ce1a9a3d2cda62, - 0xdc5c5301c56b75f7,0x7641a140cc7810fb, - 0x89b9b3e11b6329ba,0xa9e904c87fcb0a9d, - 0xac2820d9623bf429,0x546345fa9fbdcd44, - 0xd732290fbacaf133,0xa97c177947ad4095, - 0x867f59a9d4bed6c0,0x49ed8eabcccc485d, - 0xa81f301449ee8c70,0x5c68f256bfff5a74, - 0xd226fc195c6a2f8c,0x73832eec6fff3111, - 0x83585d8fd9c25db7,0xc831fd53c5ff7eab, - 0xa42e74f3d032f525,0xba3e7ca8b77f5e55, - 0xcd3a1230c43fb26f,0x28ce1bd2e55f35eb, - 0x80444b5e7aa7cf85,0x7980d163cf5b81b3, - 0xa0555e361951c366,0xd7e105bcc332621f, - 0xc86ab5c39fa63440,0x8dd9472bf3fefaa7, - 0xfa856334878fc150,0xb14f98f6f0feb951, - 0x9c935e00d4b9d8d2,0x6ed1bf9a569f33d3, - 0xc3b8358109e84f07,0xa862f80ec4700c8, - 0xf4a642e14c6262c8,0xcd27bb612758c0fa, - 0x98e7e9cccfbd7dbd,0x8038d51cb897789c, - 0xbf21e44003acdd2c,0xe0470a63e6bd56c3, - 0xeeea5d5004981478,0x1858ccfce06cac74, - 0x95527a5202df0ccb,0xf37801e0c43ebc8, - 0xbaa718e68396cffd,0xd30560258f54e6ba, - 0xe950df20247c83fd,0x47c6b82ef32a2069, - 0x91d28b7416cdd27e,0x4cdc331d57fa5441, - 0xb6472e511c81471d,0xe0133fe4adf8e952, - 0xe3d8f9e563a198e5,0x58180fddd97723a6, - 0x8e679c2f5e44ff8f,0x570f09eaa7ea7648,}; -}; -#endif // SIMDJSON_STATIC_REFLECTION - -extern SIMDJSON_DLLIMPORTEXPORT const uint64_t power_of_five_128[651*2]; -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_NUMBERPARSING_TABLES_H -/* end file simdjson/internal/numberparsing_tables.h */ -/* including simdjson/internal/simdprune_tables.h: #include "simdjson/internal/simdprune_tables.h" */ -/* begin file simdjson/internal/simdprune_tables.h */ -#ifndef SIMDJSON_INTERNAL_SIMDPRUNE_TABLES_H -#define SIMDJSON_INTERNAL_SIMDPRUNE_TABLES_H - -/* skipped duplicate #include "simdjson/base.h" */ - -#include - -namespace simdjson { // table modified and copied from -namespace internal { // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetTable - -extern SIMDJSON_DLLIMPORTEXPORT const unsigned char BitsSetTable256mul2[256]; - -extern SIMDJSON_DLLIMPORTEXPORT const uint8_t pshufb_combine_table[272]; - -// 256 * 8 bytes = 2kB, easily fits in cache. -extern SIMDJSON_DLLIMPORTEXPORT const uint64_t thintable_epi8[256]; - -} // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_INTERNAL_SIMDPRUNE_TABLES_H -/* end file simdjson/internal/simdprune_tables.h */ -#endif // SIMDJSON_GENERIC_DEPENDENCIES_H -/* end file simdjson/generic/dependencies.h */ - -/* defining SIMDJSON_CONDITIONAL_INCLUDE */ -#define SIMDJSON_CONDITIONAL_INCLUDE - -#if SIMDJSON_BUILTIN_IMPLEMENTATION_IS(arm64) -/* including simdjson/arm64/implementation.h: #include "simdjson/arm64/implementation.h" */ -/* begin file simdjson/arm64/implementation.h */ -#ifndef SIMDJSON_ARM64_IMPLEMENTATION_H -#define SIMDJSON_ARM64_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation("arm64", "ARM NEON", internal::instruction_set::NEON) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_ARM64_IMPLEMENTATION_H -/* end file simdjson/arm64/implementation.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(fallback) -/* including simdjson/fallback/implementation.h: #include "simdjson/fallback/implementation.h" */ -/* begin file simdjson/fallback/implementation.h */ -#ifndef SIMDJSON_FALLBACK_IMPLEMENTATION_H -#define SIMDJSON_FALLBACK_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation( - "fallback", - "Generic fallback implementation", - 0 - ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_FALLBACK_IMPLEMENTATION_H -/* end file simdjson/fallback/implementation.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(haswell) -/* including simdjson/haswell/implementation.h: #include "simdjson/haswell/implementation.h" */ -/* begin file simdjson/haswell/implementation.h */ -#ifndef SIMDJSON_HASWELL_IMPLEMENTATION_H -#define SIMDJSON_HASWELL_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_HASWELL -namespace simdjson { -namespace haswell { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation( - "haswell", - "Intel/AMD AVX2", - internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2 - ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_IMPLEMENTATION_H -/* end file simdjson/haswell/implementation.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(icelake) -/* including simdjson/icelake/implementation.h: #include "simdjson/icelake/implementation.h" */ -/* begin file simdjson/icelake/implementation.h */ -#ifndef SIMDJSON_ICELAKE_IMPLEMENTATION_H -#define SIMDJSON_ICELAKE_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_ICELAKE -namespace simdjson { -namespace icelake { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation( - "icelake", - "Intel/AMD AVX512", - internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2 | internal::instruction_set::AVX512F | internal::instruction_set::AVX512DQ | internal::instruction_set::AVX512CD | internal::instruction_set::AVX512BW | internal::instruction_set::AVX512VL | internal::instruction_set::AVX512VBMI2 - ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_IMPLEMENTATION_H -/* end file simdjson/icelake/implementation.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(ppc64) -/* including simdjson/ppc64/implementation.h: #include "simdjson/ppc64/implementation.h" */ -/* begin file simdjson/ppc64/implementation.h */ -#ifndef SIMDJSON_PPC64_IMPLEMENTATION_H -#define SIMDJSON_PPC64_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - -/** - * Implementation for ALTIVEC (PPC64). - */ -namespace ppc64 { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() - : simdjson::implementation("ppc64", "PPC64 ALTIVEC", - internal::instruction_set::ALTIVEC) {} - - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, size_t max_length, - std::unique_ptr &dst) - const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, - uint8_t *dst, - size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, - size_t len) const noexcept final; -}; - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_IMPLEMENTATION_H -/* end file simdjson/ppc64/implementation.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(westmere) -/* including simdjson/westmere/implementation.h: #include "simdjson/westmere/implementation.h" */ -/* begin file simdjson/westmere/implementation.h */ -#ifndef SIMDJSON_WESTMERE_IMPLEMENTATION_H -#define SIMDJSON_WESTMERE_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE -namespace simdjson { -namespace westmere { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation("westmere", "Intel/AMD SSE4.2", internal::instruction_set::SSE42 | internal::instruction_set::PCLMULQDQ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_IMPLEMENTATION_H -/* end file simdjson/westmere/implementation.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(lsx) -/* including simdjson/lsx/implementation.h: #include "simdjson/lsx/implementation.h" */ -/* begin file simdjson/lsx/implementation.h */ -#ifndef SIMDJSON_LSX_IMPLEMENTATION_H -#define SIMDJSON_LSX_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation("lsx", "LoongArch SX", internal::instruction_set::LSX) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_LSX_IMPLEMENTATION_H -/* end file simdjson/lsx/implementation.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(lasx) -/* including simdjson/lasx/implementation.h: #include "simdjson/lasx/implementation.h" */ -/* begin file simdjson/lasx/implementation.h */ -#ifndef SIMDJSON_LASX_IMPLEMENTATION_H -#define SIMDJSON_LASX_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation("lasx", "LoongArch ASX", internal::instruction_set::LASX) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_LASX_IMPLEMENTATION_H -/* end file simdjson/lasx/implementation.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(rvv_vls) -/* including simdjson/rvv-vls/implementation.h: #include "simdjson/rvv-vls/implementation.h" */ -/* begin file simdjson/rvv-vls/implementation.h */ -#ifndef SIMDJSON_RVV_VLS_IMPLEMENTATION_H -#define SIMDJSON_RVV_VLS_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { - -/** - * @private - */ -class implementation final : public simdjson::implementation { -public: - simdjson_inline implementation() : simdjson::implementation( - "rvv_vls", - "RISC-V V extension", - internal::instruction_set::RVV_VLS - ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final; -}; - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_IMPLEMENTATION_H -/* end file simdjson/rvv-vls/implementation.h */ -#else -#error Unknown SIMDJSON_BUILTIN_IMPLEMENTATION -#endif - -/* undefining SIMDJSON_CONDITIONAL_INCLUDE */ -#undef SIMDJSON_CONDITIONAL_INCLUDE - -namespace simdjson { - /** - * Function which returns a pointer to an implementation matching the "builtin" implementation. - * The builtin implementation is the best statically linked simdjson implementation that can be used by the compiling - * program. If you compile with g++ -march=haswell, this will return the haswell implementation. - * It is handy to be able to check what builtin was used: builtin_implementation()->name(). - */ - const implementation * builtin_implementation(); -} // namespace simdjson - -#endif // SIMDJSON_BUILTIN_IMPLEMENTATION_H -/* end file simdjson/builtin/implementation.h */ - -/* skipped duplicate #include "simdjson/generic/dependencies.h" */ - -/* defining SIMDJSON_CONDITIONAL_INCLUDE */ -#define SIMDJSON_CONDITIONAL_INCLUDE - -#if SIMDJSON_BUILTIN_IMPLEMENTATION_IS(arm64) -/* including simdjson/arm64.h: #include "simdjson/arm64.h" */ -/* begin file simdjson/arm64.h */ -#ifndef SIMDJSON_ARM64_H -#define SIMDJSON_ARM64_H - -/* including simdjson/arm64/begin.h: #include "simdjson/arm64/begin.h" */ -/* begin file simdjson/arm64/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "arm64" */ -#define SIMDJSON_IMPLEMENTATION arm64 -/* including simdjson/arm64/base.h: #include "simdjson/arm64/base.h" */ -/* begin file simdjson/arm64/base.h */ -#ifndef SIMDJSON_ARM64_BASE_H -#define SIMDJSON_ARM64_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * Implementation for NEON (ARMv8). - */ -namespace arm64 { - -class implementation; - -namespace { -namespace simd { -template struct simd8; -template struct simd8x64; -} // namespace simd -} // unnamed namespace - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_ARM64_BASE_H -/* end file simdjson/arm64/base.h */ -/* including simdjson/arm64/intrinsics.h: #include "simdjson/arm64/intrinsics.h" */ -/* begin file simdjson/arm64/intrinsics.h */ -#ifndef SIMDJSON_ARM64_INTRINSICS_H -#define SIMDJSON_ARM64_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This should be the correct header whether -// you use visual studio or other compilers. -#include - -static_assert(sizeof(uint8x16_t) <= simdjson::SIMDJSON_PADDING, "insufficient padding for arm64"); - -#endif // SIMDJSON_ARM64_INTRINSICS_H -/* end file simdjson/arm64/intrinsics.h */ -/* including simdjson/arm64/bitmanipulation.h: #include "simdjson/arm64/bitmanipulation.h" */ -/* begin file simdjson/arm64/bitmanipulation.h */ -#ifndef SIMDJSON_ARM64_BITMANIPULATION_H -#define SIMDJSON_ARM64_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -// We sometimes call leading_zeroes on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -// Applies only when SIMDJSON_PREFER_REVERSE_BITS is defined and true. -// (See below.) -SIMDJSON_NO_SANITIZE_UNDEFINED -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int count_ones(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); -#else - // if the system supports SVE or CSSC, __builtin_popcountll - // might be compiled to fewer single instructions. For CSSC, - // __builtin_popcountll is compiled to a single instruction. - return __builtin_popcountll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO -} - - -#if defined(__GNUC__) // catches clang and gcc -/** - * ARM has a fast 64-bit "bit reversal function" that is handy. However, - * it is not generally available as an intrinsic function under Visual - * Studio (though this might be changing). Even under clang/gcc, we - * apparently need to invoke inline assembly. - */ -/* - * We use SIMDJSON_PREFER_REVERSE_BITS as a hint that algorithms that - * work well with bit reversal may use it. - */ -#define SIMDJSON_PREFER_REVERSE_BITS 1 - -/* reverse the bits */ -simdjson_inline uint64_t reverse_bits(uint64_t input_num) { - uint64_t rev_bits; - __asm("rbit %0, %1" : "=r"(rev_bits) : "r"(input_num)); - return rev_bits; -} - -/** - * Flips bit at index 63 - lz. Thus if you have 'leading_zeroes' leading zeroes, - * then this will set to zero the leading bit. It is possible for leading_zeroes to be - * greating or equal to 63 in which case we trigger undefined behavior, but the output - * of such undefined behavior is never used. - **/ -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) { - return rev_bits ^ (uint64_t(0x8000000000000000) >> leading_zeroes); -} - -#endif - - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_ARM64_BITMANIPULATION_H -/* end file simdjson/arm64/bitmanipulation.h */ -/* including simdjson/arm64/bitmask.h: #include "simdjson/arm64/bitmask.h" */ -/* begin file simdjson/arm64/bitmask.h */ -#ifndef SIMDJSON_ARM64_BITMASK_H -#define SIMDJSON_ARM64_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { - ///////////// - // We could do this with PMULL, but it is apparently slow. - // - //#ifdef __ARM_FEATURE_CRYPTO // some ARM processors lack this extension - //return vmull_p64(-1ULL, bitmask); - //#else - // Analysis by @sebpop: - // When diffing the assembly for src/stage1_find_marks.cpp I see that the eors are all spread out - // in between other vector code, so effectively the extra cycles of the sequence do not matter - // because the GPR units are idle otherwise and the critical path is on the FP side. - // Also the PMULL requires two extra fmovs: GPR->FP (3 cycles in N1, 5 cycles in A72 ) - // and FP->GPR (2 cycles on N1 and 5 cycles on A72.) - /////////// - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; - return bitmask; -} - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif -/* end file simdjson/arm64/bitmask.h */ -/* including simdjson/arm64/numberparsing_defs.h: #include "simdjson/arm64/numberparsing_defs.h" */ -/* begin file simdjson/arm64/numberparsing_defs.h */ -#ifndef SIMDJSON_ARM64_NUMBERPARSING_DEFS_H -#define SIMDJSON_ARM64_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#if SIMDJSON_REGULAR_VISUAL_STUDIO && SIMDJSON_IS_ARM64 -// __umulh requires intrin.h -#include -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO && SIMDJSON_IS_ARM64 - -namespace simdjson { -namespace arm64 { -namespace numberparsing { - -// we don't have SSE, so let us use a scalar function -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - uint64_t val; - std::memcpy(&val, chars, sizeof(uint64_t)); - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace arm64 -} // namespace simdjson - -#ifndef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_IS_BIG_ENDIAN -#define SIMDJSON_SWAR_NUMBER_PARSING 0 -#else -#define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif -#endif - -#endif // SIMDJSON_ARM64_NUMBERPARSING_DEFS_H -/* end file simdjson/arm64/numberparsing_defs.h */ -/* including simdjson/arm64/simd.h: #include "simdjson/arm64/simd.h" */ -/* begin file simdjson/arm64/simd.h */ -#ifndef SIMDJSON_ARM64_SIMD_H -#define SIMDJSON_ARM64_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace simd { - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -namespace { -// Start of private section with Visual Studio workaround - - -#ifndef simdjson_make_uint8x16_t -#define simdjson_make_uint8x16_t(x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, \ - x13, x14, x15, x16) \ - ([=]() { \ - uint8_t array[16] = {x1, x2, x3, x4, x5, x6, x7, x8, \ - x9, x10, x11, x12, x13, x14, x15, x16}; \ - return vld1q_u8(array); \ - }()) -#endif -#ifndef simdjson_make_int8x16_t -#define simdjson_make_int8x16_t(x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, \ - x13, x14, x15, x16) \ - ([=]() { \ - int8_t array[16] = {x1, x2, x3, x4, x5, x6, x7, x8, \ - x9, x10, x11, x12, x13, x14, x15, x16}; \ - return vld1q_s8(array); \ - }()) -#endif - -#ifndef simdjson_make_uint8x8_t -#define simdjson_make_uint8x8_t(x1, x2, x3, x4, x5, x6, x7, x8) \ - ([=]() { \ - uint8_t array[8] = {x1, x2, x3, x4, x5, x6, x7, x8}; \ - return vld1_u8(array); \ - }()) -#endif -#ifndef simdjson_make_int8x8_t -#define simdjson_make_int8x8_t(x1, x2, x3, x4, x5, x6, x7, x8) \ - ([=]() { \ - int8_t array[8] = {x1, x2, x3, x4, x5, x6, x7, x8}; \ - return vld1_s8(array); \ - }()) -#endif -#ifndef simdjson_make_uint16x8_t -#define simdjson_make_uint16x8_t(x1, x2, x3, x4, x5, x6, x7, x8) \ - ([=]() { \ - uint16_t array[8] = {x1, x2, x3, x4, x5, x6, x7, x8}; \ - return vld1q_u16(array); \ - }()) -#endif -#ifndef simdjson_make_int16x8_t -#define simdjson_make_int16x8_t(x1, x2, x3, x4, x5, x6, x7, x8) \ - ([=]() { \ - int16_t array[8] = {x1, x2, x3, x4, x5, x6, x7, x8}; \ - return vld1q_s16(array); \ - }()) -#endif - -// End of private section with Visual Studio workaround -} // namespace -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO - - - template - struct simd8; - - // - // Base class of simd8 and simd8, both of which use uint8x16_t internally. - // - template> - struct base_u8 { - uint8x16_t value; - static const int SIZE = sizeof(value); - - // Conversion from/to SIMD register - simdjson_inline base_u8(const uint8x16_t _value) : value(_value) {} - simdjson_inline operator const uint8x16_t&() const { return this->value; } - simdjson_inline operator uint8x16_t&() { return this->value; } - - // Bit operations - simdjson_inline simd8 operator|(const simd8 other) const { return vorrq_u8(*this, other); } - simdjson_inline simd8 operator&(const simd8 other) const { return vandq_u8(*this, other); } - simdjson_inline simd8 operator^(const simd8 other) const { return veorq_u8(*this, other); } - simdjson_inline simd8 bit_andnot(const simd8 other) const { return vbicq_u8(*this, other); } - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - simdjson_inline simd8& operator|=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline simd8& operator&=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline simd8& operator^=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast ^ other; return *this_cast; } - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return vceqq_u8(lhs, rhs); } - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return vextq_u8(prev_chunk, *this, 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base_u8 { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - static simdjson_inline simd8 splat(bool _value) { return vmovq_n_u8(uint8_t(-(!!_value))); } - - simdjson_inline simd8(const uint8x16_t _value) : base_u8(_value) {} - // False constructor - simdjson_inline simd8() : simd8(vdupq_n_u8(0)) {} - // Splat constructor - simdjson_inline simd8(bool _value) : simd8(splat(_value)) {} - - // We return uint32_t instead of uint16_t because that seems to be more efficient for most - // purposes (cutting it down to uint16_t costs performance in some compilers). - simdjson_inline uint32_t to_bitmask() const { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - const uint8x16_t bit_mask = simdjson_make_uint8x16_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80); -#else - const uint8x16_t bit_mask = {0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80}; -#endif - auto minput = *this & bit_mask; - uint8x16_t tmp = vpaddq_u8(minput, minput); - tmp = vpaddq_u8(tmp, tmp); - tmp = vpaddq_u8(tmp, tmp); - return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); - } - // Returns 4-bit out of each byte, alternating between the high 4 bits and low - // bits result it is 64 bit. - simdjson_inline uint64_t to_bitmask64() const { - return vget_lane_u64( - vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); - } - // Integer umaxv, not a float compare: flush-to-zero would treat a denormal as zero. - simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } - }; - - // Unsigned bytes - template<> - struct simd8: base_u8 { - static simdjson_inline uint8x16_t splat(uint8_t _value) { return vmovq_n_u8(_value); } - static simdjson_inline uint8x16_t zero() { return vdupq_n_u8(0); } - static simdjson_inline uint8x16_t load(const uint8_t* values) { return vld1q_u8(values); } - - simdjson_inline simd8(const uint8x16_t _value) : base_u8(_value) {} - // Zero constructor - simdjson_inline simd8() : simd8(zero()) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Member-by-member initialization -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(simdjson_make_uint8x16_t( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} -#else - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(uint8x16_t{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} -#endif - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Store to array - simdjson_inline void store(uint8_t dst[16]) const { return vst1q_u8(dst, *this); } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return vqaddq_u8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return vqsubq_u8(*this, other); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return vaddq_u8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return vsubq_u8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } - - // Order-specific operations - simdjson_inline uint8_t max_val() const { return vmaxvq_u8(*this); } - simdjson_inline uint8_t min_val() const { return vminvq_u8(*this); } - simdjson_inline simd8 max_val(const simd8 other) const { return vmaxq_u8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return vminq_u8(*this, other); } - simdjson_inline simd8 operator<=(const simd8 other) const { return vcleq_u8(*this, other); } - simdjson_inline simd8 operator>=(const simd8 other) const { return vcgeq_u8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return vcltq_u8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return vcgtq_u8(*this, other); } - // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. For ARM, returns all 1's. - simdjson_inline simd8 gt_bits(const simd8 other) const { return simd8(*this > other); } - // Same as <, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. For ARM, returns all 1's. - simdjson_inline simd8 lt_bits(const simd8 other) const { return simd8(*this < other); } - - // Bit-specific operations - simdjson_inline simd8 any_bits_set(simd8 bits) const { return vtstq_u8(*this, bits); } - simdjson_inline bool any_bits_set_anywhere() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } - template - simdjson_inline simd8 shr() const { return vshrq_n_u8(*this, N); } - template - simdjson_inline simd8 shl() const { return vshlq_n_u8(*this, N); } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return lookup_table.apply_lookup_16_to(*this); - } - - // Returns 4-bit out of each byte, alternating between the high 4 bits and low - // bits result it is 64 bit. - simdjson_inline uint64_t to_bitmask64() const { - return vget_lane_u64( - vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); - } - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint16_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - uint64x2_t shufmask64 = {thintable_epi8[mask1], thintable_epi8[mask2]}; - uint8x16_t shufmask = vreinterpretq_u8_u64(shufmask64); - // we increment by 0x08 the second half of the mask -#if SIMDJSON_REGULAR_VISUAL_STUDIO - uint8x16_t inc = simdjson_make_uint8x16_t(0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08); -#else - uint8x16_t inc = {0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08}; -#endif - shufmask = vaddq_u8(shufmask, inc); - // this is the version "nearly pruned" - uint8x16_t pruned = vqtbl1q_u8(*this, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - uint8x16_t compactmask = vld1q_u8(reinterpret_cast(pshufb_combine_table + pop1 * 8)); - uint8x16_t answer = vqtbl1q_u8(pruned, compactmask); - vst1q_u8(reinterpret_cast(output), answer); - } - - // Copies all bytes corresponding to a 0 in the low half of the mask (interpreted as a - // bitset) to output1, then those corresponding to a 0 in the high half to output2. - template - simdjson_inline void compress_halves(uint16_t mask, L *output1, L *output2) const { - using internal::thintable_epi8; - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - uint8x8_t compactmask1 = vcreate_u8(thintable_epi8[mask1]); - uint8x8_t compactmask2 = vcreate_u8(thintable_epi8[mask2]); - // we increment by 0x08 the second half of the mask -#if SIMDJSON_REGULAR_VISUAL_STUDIO - uint8x8_t inc = simdjson_make_uint8x8_t(0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08); -#else - uint8x8_t inc = {0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08}; -#endif - compactmask2 = vadd_u8(compactmask2, inc); - // store each result (with the second store possibly overlapping the first) - vst1_u8((uint8_t*)output1, vqtbl1_u8(*this, compactmask1)); - vst1_u8((uint8_t*)output2, vqtbl1_u8(*this, compactmask2)); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - - template - simdjson_inline simd8 apply_lookup_16_to(const simd8 original) { - return vqtbl1q_u8(*this, simd8(original)); - } - }; - - // Signed bytes - template<> - struct simd8 { - int8x16_t value; - - static simdjson_inline simd8 splat(int8_t _value) { return vmovq_n_s8(_value); } - static simdjson_inline simd8 zero() { return vdupq_n_s8(0); } - static simdjson_inline simd8 load(const int8_t values[16]) { return vld1q_s8(values); } - - // Conversion from/to SIMD register - simdjson_inline simd8(const int8x16_t _value) : value{_value} {} - simdjson_inline operator const int8x16_t&() const { return this->value; } - simdjson_inline operator int8x16_t&() { return this->value; } - - // Zero constructor - simdjson_inline simd8() : simd8(zero()) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} - // Member-by-member initialization -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(simdjson_make_int8x16_t( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} -#else - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(int8x16_t{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} -#endif - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Store to array - simdjson_inline void store(int8_t dst[16]) const { return vst1q_s8(dst, *this); } - - // Explicit conversion to/from unsigned - // - // Under Visual Studio/ARM64 uint8x16_t and int8x16_t are apparently the same type. - // In theory, we could check this occurrence with std::same_as and std::enabled_if but it is C++14 - // and relatively ugly and hard to read. -#ifndef SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline explicit simd8(const uint8x16_t other): simd8(vreinterpretq_s8_u8(other)) {} -#endif - simdjson_inline explicit operator simd8() const { return vreinterpretq_u8_s8(this->value); } - - // Math - simdjson_inline simd8 operator+(const simd8 other) const { return vaddq_s8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return vsubq_s8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return vmaxq_s8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return vminq_s8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return vcgtq_s8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return vcltq_s8(*this, other); } - simdjson_inline simd8 operator==(const simd8 other) const { return vceqq_s8(*this, other); } - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return vextq_s8(prev_chunk, *this, 16 - N); - } - - // Perform a lookup assuming no value is larger than 16 - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return lookup_table.apply_lookup_16_to(*this); - } - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - - template - simdjson_inline simd8 apply_lookup_16_to(const simd8 original) { - return vqtbl1q_s8(*this, simd8(original)); - } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "ARM kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - this->chunks[2].store(ptr+sizeof(simd8)*2); - this->chunks[3].store(ptr+sizeof(simd8)*3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - uint64_t popcounts = vget_lane_u64(vreinterpret_u64_u8(vcnt_u8(vcreate_u8(~mask))), 0); - // compute the prefix sum of the popcounts of each byte - uint64_t offsets = popcounts * 0x0101010101010101; - this->chunks[0].compress_halves(uint16_t(mask), output, &output[popcounts & 0xFF]); - this->chunks[1].compress_halves(uint16_t(mask >> 16), &output[(offsets >> 8) & 0xFF], &output[(offsets >> 16) & 0xFF]); - this->chunks[2].compress_halves(uint16_t(mask >> 32), &output[(offsets >> 24) & 0xFF], &output[(offsets >> 32) & 0xFF]); - this->chunks[3].compress_halves(uint16_t(mask >> 48), &output[(offsets >> 40) & 0xFF], &output[(offsets >> 48) & 0xFF]); - return offsets >> 56; - } - - simdjson_inline uint64_t to_bitmask() const { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - const uint8x16_t bit_mask = simdjson_make_uint8x16_t( - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80 - ); -#else - const uint8x16_t bit_mask = { - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, - 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80 - }; -#endif - // Add each of the elements next to each other, successively, to stuff each 8 byte mask into one. - uint8x16_t sum0 = vpaddq_u8(this->chunks[0] & bit_mask, this->chunks[1] & bit_mask); - uint8x16_t sum1 = vpaddq_u8(this->chunks[2] & bit_mask, this->chunks[3] & bit_mask); - sum0 = vpaddq_u8(sum0, sum1); - sum0 = vpaddq_u8(sum0, sum0); - return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_ARM64_SIMD_H -/* end file simdjson/arm64/simd.h */ -/* including simdjson/arm64/stringparsing_defs.h: #include "simdjson/arm64/stringparsing_defs.h" */ -/* begin file simdjson/arm64/stringparsing_defs.h */ -#ifndef SIMDJSON_ARM64_STRINGPARSING_DEFS_H -#define SIMDJSON_ARM64_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + sizeof(v0)); - v0.store(dst); - v1.store(dst + sizeof(v0)); - - // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on ARM; therefore, we - // smash them together into a 64-byte mask and get the bitmask from there. - uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; -} - - -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_ARM64_STRINGPARSING_DEFS_H -/* end file simdjson/arm64/stringparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 -/* end file simdjson/arm64/begin.h */ -/* including simdjson/generic/amalgamated.h for arm64: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for arm64 */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for arm64: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for arm64 */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for arm64 */ -/* including simdjson/generic/jsoncharutils.h for arm64: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for arm64 */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for arm64 */ -/* including simdjson/generic/atomparsing.h for arm64: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for arm64 */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace arm64 { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for arm64 */ -/* including simdjson/generic/dom_parser_implementation.h for arm64: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for arm64 */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace arm64 -} // namespace simdjson - -namespace simdjson { -namespace arm64 { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for arm64 */ -/* including simdjson/generic/implementation_simdjson_result_base.h for arm64: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for arm64 */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for arm64 */ -/* including simdjson/generic/numberparsing.h for arm64: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for arm64 */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace arm64 { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for arm64 */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for arm64: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for arm64 */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for arm64 */ -/* end file simdjson/generic/amalgamated.h for arm64 */ -/* including simdjson/arm64/end.h: #include "simdjson/arm64/end.h" */ -/* begin file simdjson/arm64/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT -/* undefining SIMDJSON_IMPLEMENTATION from "arm64" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/arm64/end.h */ - -#endif // SIMDJSON_ARM64_H -/* end file simdjson/arm64.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(fallback) -/* including simdjson/fallback.h: #include "simdjson/fallback.h" */ -/* begin file simdjson/fallback.h */ -#ifndef SIMDJSON_FALLBACK_H -#define SIMDJSON_FALLBACK_H - -/* including simdjson/fallback/begin.h: #include "simdjson/fallback/begin.h" */ -/* begin file simdjson/fallback/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "fallback" */ -#define SIMDJSON_IMPLEMENTATION fallback -/* including simdjson/fallback/base.h: #include "simdjson/fallback/base.h" */ -/* begin file simdjson/fallback/base.h */ -#ifndef SIMDJSON_FALLBACK_BASE_H -#define SIMDJSON_FALLBACK_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * Fallback implementation (runs on any machine). - */ -namespace fallback { - -class implementation; - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_FALLBACK_BASE_H -/* end file simdjson/fallback/base.h */ -/* including simdjson/fallback/bitmanipulation.h: #include "simdjson/fallback/bitmanipulation.h" */ -/* begin file simdjson/fallback/bitmanipulation.h */ -#ifndef SIMDJSON_FALLBACK_BITMANIPULATION_H -#define SIMDJSON_FALLBACK_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { -namespace { - -#if defined(_MSC_VER) && !defined(_M_ARM64) && !defined(_M_X64) -static inline unsigned char _BitScanForward64(unsigned long* ret, uint64_t x) { - unsigned long x0 = (unsigned long)x, top, bottom; - _BitScanForward(&top, (unsigned long)(x >> 32)); - _BitScanForward(&bottom, x0); - *ret = x0 ? bottom : 32 + top; - return x != 0; -} -static unsigned char _BitScanReverse64(unsigned long* ret, uint64_t x) { - unsigned long x1 = (unsigned long)(x >> 32), top, bottom; - _BitScanReverse(&top, x1); - _BitScanReverse(&bottom, (unsigned long)x); - *ret = x1 ? top + 32 : bottom; - return x != 0; -} -#endif - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { -#ifdef _MSC_VER - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// _MSC_VER -} - -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - // Search the mask data from least significant bit (LSB) - // to most significant bit (MSB) for a set bit (1). - if (_BitScanForward64(&trailing_zero, input_num)) - return (int)trailing_zero; - else return 64; -#else - return __builtin_ctzll(input_num); -#endif// _MSC_VER -} - -} // unnamed namespace -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_FALLBACK_BITMANIPULATION_H -/* end file simdjson/fallback/bitmanipulation.h */ -/* including simdjson/fallback/stringparsing_defs.h: #include "simdjson/fallback/stringparsing_defs.h" */ -/* begin file simdjson/fallback/stringparsing_defs.h */ -#ifndef SIMDJSON_FALLBACK_STRINGPARSING_DEFS_H -#define SIMDJSON_FALLBACK_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { -namespace { - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 1; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return c == '"'; } - simdjson_inline bool has_backslash() { return c == '\\'; } - simdjson_inline int quote_index() { return c == '"' ? 0 : 1; } - simdjson_inline int backslash_index() { return c == '\\' ? 0 : 1; } - - uint8_t c; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // store to dest unconditionally - we can overwrite the bits we don't like later - dst[0] = src[0]; - return { src[0] }; -} - - -} // unnamed namespace -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_FALLBACK_STRINGPARSING_DEFS_H -/* end file simdjson/fallback/stringparsing_defs.h */ -/* including simdjson/fallback/numberparsing_defs.h: #include "simdjson/fallback/numberparsing_defs.h" */ -/* begin file simdjson/fallback/numberparsing_defs.h */ -#ifndef SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H -#define SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#ifdef JSON_TEST_NUMBERS // for unit testing -void found_invalid_number(const uint8_t *buf); -void found_integer(int64_t result, const uint8_t *buf); -void found_unsigned_integer(uint64_t result, const uint8_t *buf); -void found_float(double result, const uint8_t *buf); -#endif - -namespace simdjson { -namespace fallback { -namespace numberparsing { - -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const char *chars) { - uint64_t val; - memcpy(&val, chars, sizeof(uint64_t)); - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - return parse_eight_digits_unrolled(reinterpret_cast(chars)); -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace fallback -} // namespace simdjson - -#ifndef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_IS_BIG_ENDIAN -#define SIMDJSON_SWAR_NUMBER_PARSING 0 -#else -#define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif -#endif - -#endif // SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H -/* end file simdjson/fallback/numberparsing_defs.h */ -/* end file simdjson/fallback/begin.h */ -/* including simdjson/generic/amalgamated.h for fallback: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for fallback */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for fallback: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for fallback */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for fallback */ -/* including simdjson/generic/jsoncharutils.h for fallback: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for fallback */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for fallback */ -/* including simdjson/generic/atomparsing.h for fallback: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for fallback */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace fallback { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for fallback */ -/* including simdjson/generic/dom_parser_implementation.h for fallback: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for fallback */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace fallback -} // namespace simdjson - -namespace simdjson { -namespace fallback { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for fallback */ -/* including simdjson/generic/implementation_simdjson_result_base.h for fallback: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for fallback */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for fallback */ -/* including simdjson/generic/numberparsing.h for fallback: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for fallback */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace fallback { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for fallback */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for fallback: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for fallback */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace fallback { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace fallback -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for fallback */ -/* end file simdjson/generic/amalgamated.h for fallback */ -/* including simdjson/fallback/end.h: #include "simdjson/fallback/end.h" */ -/* begin file simdjson/fallback/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* undefining SIMDJSON_IMPLEMENTATION from "fallback" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/fallback/end.h */ - -#endif // SIMDJSON_FALLBACK_H -/* end file simdjson/fallback.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(haswell) -/* including simdjson/haswell.h: #include "simdjson/haswell.h" */ -/* begin file simdjson/haswell.h */ -#ifndef SIMDJSON_HASWELL_H -#define SIMDJSON_HASWELL_H - -/* including simdjson/haswell/begin.h: #include "simdjson/haswell/begin.h" */ -/* begin file simdjson/haswell/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "haswell" */ -#define SIMDJSON_IMPLEMENTATION haswell - -/* including simdjson/haswell/base.h: #include "simdjson/haswell/base.h" */ -/* begin file simdjson/haswell/base.h */ -#ifndef SIMDJSON_HASWELL_BASE_H -#define SIMDJSON_HASWELL_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_HASWELL -namespace simdjson { -/** - * Implementation for Haswell (Intel AVX2). - */ -namespace haswell { - -class implementation; - -namespace { -namespace simd { -template struct simd8; -template struct simd8x64; -} // namespace simd -} // unnamed namespace - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_BASE_H -/* end file simdjson/haswell/base.h */ -/* including simdjson/haswell/intrinsics.h: #include "simdjson/haswell/intrinsics.h" */ -/* begin file simdjson/haswell/intrinsics.h */ -#ifndef SIMDJSON_HASWELL_INTRINSICS_H -#define SIMDJSON_HASWELL_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - * e.g., if __AVX2__ is set... in turn, we normally set these - * macros by compiling against the corresponding architecture - * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole - * software with these advanced instructions. In simdjson, we - * want to compile the whole program for a generic target, - * and only target our specific kernels. As a workaround, - * we directly include the needed headers. These headers would - * normally guard against such usage, but we carefully included - * (or ) before, so the headers - * are fooled. - */ -#include // for _blsr_u64 -#include // for __lzcnt64 -#include // for most things (AVX2, AVX512, _popcnt64) -#include -#include -#include -#include -#include // for _mm_clmulepi64_si128 -// unfortunately, we may not get _blsr_u64, but, thankfully, clang -// has it as a macro. -#ifndef _blsr_u64 -// we roll our own -#define _blsr_u64(n) ((n - 1) & n) -#endif // _blsr_u64 -#endif // SIMDJSON_CLANG_VISUAL_STUDIO - -static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for haswell kernel."); - -#endif // SIMDJSON_HASWELL_INTRINSICS_H -/* end file simdjson/haswell/intrinsics.h */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_HASWELL -// We enable bmi2 only if LLVM/clang is used, because GCC may not -// make good use of it. See https://github.com/simdjson/simdjson/pull/2243 -#if defined(__clang__) -SIMDJSON_TARGET_REGION("avx2,bmi,bmi2,pclmul,lzcnt,popcnt") -#else -SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt,popcnt") -#endif -#endif - -/* including simdjson/haswell/bitmanipulation.h: #include "simdjson/haswell/bitmanipulation.h" */ -/* begin file simdjson/haswell/bitmanipulation.h */ -#ifndef SIMDJSON_HASWELL_BITMANIPULATION_H -#define SIMDJSON_HASWELL_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmask.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return (int)_tzcnt_u64(input_num); -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - //////// - // You might expect the next line to be equivalent to - // return (int)_tzcnt_u64(input_num); - // but the generated code differs and might be less efficient? - //////// - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return _blsr_u64(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { - return int(_lzcnt_u64(input_num)); -} - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num);// Visual Studio wants two underscores -} -#else -simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); -} -#endif - - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_BITMANIPULATION_H -/* end file simdjson/haswell/bitmanipulation.h */ -/* including simdjson/haswell/bitmask.h: #include "simdjson/haswell/bitmask.h" */ -/* begin file simdjson/haswell/bitmask.h */ -#ifndef SIMDJSON_HASWELL_BITMASK_H -#define SIMDJSON_HASWELL_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { - // There should be no such thing with a processor supporting avx2 - // but not clmul. - __m128i all_ones = _mm_set1_epi8('\xFF'); - __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); - return _mm_cvtsi128_si64(result); -} - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_BITMASK_H -/* end file simdjson/haswell/bitmask.h */ -/* including simdjson/haswell/numberparsing_defs.h: #include "simdjson/haswell/numberparsing_defs.h" */ -/* begin file simdjson/haswell/numberparsing_defs.h */ -#ifndef SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H -#define SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace numberparsing { - -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - // this actually computes *16* values so we are being wasteful. - const __m128i ascii0 = _mm_set1_epi8('0'); - const __m128i mul_1_10 = - _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); - const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); - const __m128i mul_1_10000 = - _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); - const __m128i input = _mm_sub_epi8( - _mm_loadu_si128(reinterpret_cast(chars)), ascii0); - const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); - const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); - const __m128i t3 = _mm_packus_epi32(t2, t2); - const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); - return _mm_cvtsi128_si32( - t4); // only captures the sum of the first 8 digits, drop the rest -} - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace haswell -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H -/* end file simdjson/haswell/numberparsing_defs.h */ -/* including simdjson/haswell/simd.h: #include "simdjson/haswell/simd.h" */ -/* begin file simdjson/haswell/simd.h */ -#ifndef SIMDJSON_HASWELL_SIMD_H -#define SIMDJSON_HASWELL_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m256i value; - - // Zero constructor - simdjson_inline base() : value{__m256i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m256i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m256i&() const { return this->value; } - simdjson_inline operator __m256i&() { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm256_or_si256(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm256_and_si256(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8: base> { - typedef uint32_t bitmask_t; - typedef uint64_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m256i _value) : base>(_value) {} - - friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm256_cmpeq_epi8(lhs, rhs); } - - static const int SIZE = sizeof(base::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return _mm256_alignr_epi8(*this, _mm256_permute2x128_si256(prev_chunk, *this, 0x21), 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m256i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return _mm256_movemask_epi8(*this); } - simdjson_inline bool any() const { return !_mm256_testz_si256(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { return _mm256_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm256_setzero_si256(); } - static simdjson_inline simd8 load(const T values[32]) { - return _mm256_loadu_si256(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m256i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[32]) const { return _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), *this); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm256_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm256_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm256_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 32 - count_ones(mask) bytes of the result are significant but 32 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint32_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in four steps, first 8 bytes and then second 8 bytes... - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits - uint8_t mask3 = uint8_t(mask >> 16); // ... - uint8_t mask4 = uint8_t(mask >> 24); // ... - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - __m256i shufmask = _mm256_set_epi64x(thintable_epi8[mask4], thintable_epi8[mask3], - thintable_epi8[mask2], thintable_epi8[mask1]); - // we increment by 0x08 the second half of the mask and so forth - shufmask = - _mm256_add_epi8(shufmask, _mm256_set_epi32(0x18181818, 0x18181818, - 0x10101010, 0x10101010, 0x08080808, 0x08080808, 0, 0)); - // this is the version "nearly pruned" - __m256i pruned = _mm256_shuffle_epi8(*this, shufmask); - // we still need to put the pieces back together. - // we compute the popcount of the first words: - int pop1 = BitsSetTable256mul2[mask1]; - int pop3 = BitsSetTable256mul2[mask3]; - - // then load the corresponding mask - // could be done with _mm256_loadu2_m128i but many standard libraries omit this intrinsic. - __m256i v256 = _mm256_castsi128_si256( - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8))); - __m256i compactmask = _mm256_insertf128_si256(v256, - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop3 * 8)), 1); - __m256i almostthere = _mm256_shuffle_epi8(pruned, compactmask); - // We just need to write out the result. - // This is the tricky bit that is hard to do - // if we want to return a SIMD register, since there - // is no single-instruction approach to recombine - // the two 128-bit lanes with an offset. - __m128i v128; - v128 = _mm256_castsi256_si128(almostthere); - _mm_storeu_si128( reinterpret_cast<__m128i *>(output), v128); - v128 = _mm256_extractf128_si256(almostthere, 1); - _mm_storeu_si128( reinterpret_cast<__m128i *>(output + 16 - count_ones(mask & 0xFFFF)), v128); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, - int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, - int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31 - ) : simd8(_mm256_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v16,v17,v18,v19,v20,v21,v22,v23, - v24,v25,v26,v27,v28,v29,v30,v31 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epi8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return _mm256_cmpgt_epi8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm256_cmpgt_epi8(other, *this); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, - uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, - uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31 - ) : simd8(_mm256_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v16,v17,v18,v19,v20,v21,v22,v23, - v24,v25,v26,v27,v28,v29,v30,v31 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm256_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm256_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epu8(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7-N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 2, "Haswell kernel should use two registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+32)} {} - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - uint32_t mask1 = uint32_t(mask); - uint32_t mask2 = uint32_t(mask >> 32); - this->chunks[0].compress(mask1, output); - this->chunks[1].compress(mask2, output + 32 - count_ones(mask1)); - return 64 - count_ones(mask); - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r_hi = this->chunks[1].to_bitmask(); - return r_lo | (r_hi << 32); - } - - simdjson_inline simd8 reduce_or() const { - return this->chunks[0] | this->chunks[1]; - } - - simdjson_inline simd8x64 bit_or(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] | mask, - this->chunks[1] | mask - ); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_SIMD_H -/* end file simdjson/haswell/simd.h */ -/* including simdjson/haswell/stringparsing_defs.h: #include "simdjson/haswell/stringparsing_defs.h" */ -/* begin file simdjson/haswell/stringparsing_defs.h */ -#ifndef SIMDJSON_HASWELL_STRINGPARSING_DEFS_H -#define SIMDJSON_HASWELL_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 15 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - // store to dest unconditionally - we can overwrite the bits we don't like later - v.store(dst); - return { - static_cast((v == '\\').to_bitmask()), // bs_bits - static_cast((v == '"').to_bitmask()), // quote_bits - }; -} - - -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_STRINGPARSING_DEFS_H -/* end file simdjson/haswell/stringparsing_defs.h */ -/* end file simdjson/haswell/begin.h */ -/* including simdjson/generic/amalgamated.h for haswell: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for haswell */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for haswell: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for haswell */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for haswell */ -/* including simdjson/generic/jsoncharutils.h for haswell: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for haswell */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for haswell */ -/* including simdjson/generic/atomparsing.h for haswell: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for haswell */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace haswell { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for haswell */ -/* including simdjson/generic/dom_parser_implementation.h for haswell: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for haswell */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace haswell -} // namespace simdjson - -namespace simdjson { -namespace haswell { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for haswell */ -/* including simdjson/generic/implementation_simdjson_result_base.h for haswell: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for haswell */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for haswell */ -/* including simdjson/generic/numberparsing.h for haswell: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for haswell */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace haswell { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for haswell */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for haswell: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for haswell */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for haswell */ -/* end file simdjson/generic/amalgamated.h for haswell */ -/* including simdjson/haswell/end.h: #include "simdjson/haswell/end.h" */ -/* begin file simdjson/haswell/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_HASWELL -SIMDJSON_UNTARGET_REGION -#endif - -/* undefining SIMDJSON_IMPLEMENTATION from "haswell" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/haswell/end.h */ - -#endif // SIMDJSON_HASWELL_H -/* end file simdjson/haswell.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(icelake) -/* including simdjson/icelake.h: #include "simdjson/icelake.h" */ -/* begin file simdjson/icelake.h */ -#ifndef SIMDJSON_ICELAKE_H -#define SIMDJSON_ICELAKE_H - -/* including simdjson/icelake/begin.h: #include "simdjson/icelake/begin.h" */ -/* begin file simdjson/icelake/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "icelake" */ -#define SIMDJSON_IMPLEMENTATION icelake -/* including simdjson/icelake/base.h: #include "simdjson/icelake/base.h" */ -/* begin file simdjson/icelake/base.h */ -#ifndef SIMDJSON_ICELAKE_BASE_H -#define SIMDJSON_ICELAKE_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_ICELAKE -namespace simdjson { -/** - * Implementation for Icelake (Intel AVX512). - */ -namespace icelake { - -class implementation; - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_BASE_H -/* end file simdjson/icelake/base.h */ -/* including simdjson/icelake/intrinsics.h: #include "simdjson/icelake/intrinsics.h" */ -/* begin file simdjson/icelake/intrinsics.h */ -#ifndef SIMDJSON_ICELAKE_INTRINSICS_H -#define SIMDJSON_ICELAKE_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - * e.g., if __AVX2__ is set... in turn, we normally set these - * macros by compiling against the corresponding architecture - * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole - * software with these advanced instructions. In simdjson, we - * want to compile the whole program for a generic target, - * and only target our specific kernels. As a workaround, - * we directly include the needed headers. These headers would - * normally guard against such usage, but we carefully included - * (or ) before, so the headers - * are fooled. - */ -#include // for _blsr_u64 -#include // for __lzcnt64 -#include // for most things (AVX2, AVX512, _popcnt64) -#include -#include -#include -#include -#include // for _mm_clmulepi64_si128 -// Important: we need the AVX-512 headers: -#include -#include -#include -#include -#include -#include -#include -// unfortunately, we may not get _blsr_u64, but, thankfully, clang -// has it as a macro. -#ifndef _blsr_u64 -// we roll our own -#define _blsr_u64(n) ((n - 1) & n) -#endif // _blsr_u64 -#endif // SIMDJSON_CLANG_VISUAL_STUDIO - -static_assert(sizeof(__m512i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for icelake"); - -#endif // SIMDJSON_ICELAKE_INTRINSICS_H -/* end file simdjson/icelake/intrinsics.h */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_ICELAKE -SIMDJSON_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi,avx512vbmi2,avx512vl,avx2,bmi,pclmul,lzcnt,popcnt") -#endif - -/* including simdjson/icelake/bitmanipulation.h: #include "simdjson/icelake/bitmanipulation.h" */ -/* begin file simdjson/icelake/bitmanipulation.h */ -#ifndef SIMDJSON_ICELAKE_BITMANIPULATION_H -#define SIMDJSON_ICELAKE_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return (int)_tzcnt_u64(input_num); -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - //////// - // You might expect the next line to be equivalent to - // return (int)_tzcnt_u64(input_num); - // but the generated code differs and might be less efficient? - //////// - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return _blsr_u64(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { - return int(_lzcnt_u64(input_num)); -} - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows - return __popcnt64(input_num);// Visual Studio wants two underscores -} -#else -simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); -} -#endif - - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_BITMANIPULATION_H -/* end file simdjson/icelake/bitmanipulation.h */ -/* including simdjson/icelake/bitmask.h: #include "simdjson/icelake/bitmask.h" */ -/* begin file simdjson/icelake/bitmask.h */ -#ifndef SIMDJSON_ICELAKE_BITMASK_H -#define SIMDJSON_ICELAKE_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { - // There should be no such thing with a processor supporting avx2 - // but not clmul. - __m128i all_ones = _mm_set1_epi8('\xFF'); - __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); - return _mm_cvtsi128_si64(result); -} - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_BITMASK_H -/* end file simdjson/icelake/bitmask.h */ -/* including simdjson/icelake/simd.h: #include "simdjson/icelake/simd.h" */ -/* begin file simdjson/icelake/simd.h */ -#ifndef SIMDJSON_ICELAKE_SIMD_H -#define SIMDJSON_ICELAKE_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if defined(__GNUC__) && !defined(__clang__) -#if __GNUC__ == 8 -#define SIMDJSON_GCC8 1 -#endif // __GNUC__ == 8 -#endif // defined(__GNUC__) && !defined(__clang__) - -#if SIMDJSON_GCC8 -/** - * GCC 8 fails to provide _mm512_set_epi8. We roll our own. - */ -inline __m512i _mm512_set_epi8(uint8_t a0, uint8_t a1, uint8_t a2, uint8_t a3, uint8_t a4, uint8_t a5, uint8_t a6, uint8_t a7, uint8_t a8, uint8_t a9, uint8_t a10, uint8_t a11, uint8_t a12, uint8_t a13, uint8_t a14, uint8_t a15, uint8_t a16, uint8_t a17, uint8_t a18, uint8_t a19, uint8_t a20, uint8_t a21, uint8_t a22, uint8_t a23, uint8_t a24, uint8_t a25, uint8_t a26, uint8_t a27, uint8_t a28, uint8_t a29, uint8_t a30, uint8_t a31, uint8_t a32, uint8_t a33, uint8_t a34, uint8_t a35, uint8_t a36, uint8_t a37, uint8_t a38, uint8_t a39, uint8_t a40, uint8_t a41, uint8_t a42, uint8_t a43, uint8_t a44, uint8_t a45, uint8_t a46, uint8_t a47, uint8_t a48, uint8_t a49, uint8_t a50, uint8_t a51, uint8_t a52, uint8_t a53, uint8_t a54, uint8_t a55, uint8_t a56, uint8_t a57, uint8_t a58, uint8_t a59, uint8_t a60, uint8_t a61, uint8_t a62, uint8_t a63) { - return _mm512_set_epi64(uint64_t(a7) + (uint64_t(a6) << 8) + (uint64_t(a5) << 16) + (uint64_t(a4) << 24) + (uint64_t(a3) << 32) + (uint64_t(a2) << 40) + (uint64_t(a1) << 48) + (uint64_t(a0) << 56), - uint64_t(a15) + (uint64_t(a14) << 8) + (uint64_t(a13) << 16) + (uint64_t(a12) << 24) + (uint64_t(a11) << 32) + (uint64_t(a10) << 40) + (uint64_t(a9) << 48) + (uint64_t(a8) << 56), - uint64_t(a23) + (uint64_t(a22) << 8) + (uint64_t(a21) << 16) + (uint64_t(a20) << 24) + (uint64_t(a19) << 32) + (uint64_t(a18) << 40) + (uint64_t(a17) << 48) + (uint64_t(a16) << 56), - uint64_t(a31) + (uint64_t(a30) << 8) + (uint64_t(a29) << 16) + (uint64_t(a28) << 24) + (uint64_t(a27) << 32) + (uint64_t(a26) << 40) + (uint64_t(a25) << 48) + (uint64_t(a24) << 56), - uint64_t(a39) + (uint64_t(a38) << 8) + (uint64_t(a37) << 16) + (uint64_t(a36) << 24) + (uint64_t(a35) << 32) + (uint64_t(a34) << 40) + (uint64_t(a33) << 48) + (uint64_t(a32) << 56), - uint64_t(a47) + (uint64_t(a46) << 8) + (uint64_t(a45) << 16) + (uint64_t(a44) << 24) + (uint64_t(a43) << 32) + (uint64_t(a42) << 40) + (uint64_t(a41) << 48) + (uint64_t(a40) << 56), - uint64_t(a55) + (uint64_t(a54) << 8) + (uint64_t(a53) << 16) + (uint64_t(a52) << 24) + (uint64_t(a51) << 32) + (uint64_t(a50) << 40) + (uint64_t(a49) << 48) + (uint64_t(a48) << 56), - uint64_t(a63) + (uint64_t(a62) << 8) + (uint64_t(a61) << 16) + (uint64_t(a60) << 24) + (uint64_t(a59) << 32) + (uint64_t(a58) << 40) + (uint64_t(a57) << 48) + (uint64_t(a56) << 56)); -} -#endif // SIMDJSON_GCC8 - - - -namespace simdjson { -namespace icelake { -namespace { -namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m512i value; - - // Zero constructor - simdjson_inline base() : value{__m512i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m512i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m512i&() const { return this->value; } - simdjson_inline operator __m512i&() { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm512_or_si512(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm512_and_si512(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm512_xor_si512(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm512_andnot_si512(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8: base> { - typedef uint32_t bitmask_t; - typedef uint64_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m512i _value) : base>(_value) {} - - friend simdjson_really_inline uint64_t operator==(const simd8 lhs, const simd8 rhs) { - return _mm512_cmpeq_epi8_mask(lhs, rhs); - } - static const int SIZE = sizeof(base::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - // workaround for compilers unable to figure out that 16 - N is a constant (GCC 8) - constexpr int shift = 16 - N; - return _mm512_alignr_epi8(*this, _mm512_permutex2var_epi64(prev_chunk, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), *this), shift); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm512_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m512i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - simdjson_inline bool any() const { return !!_mm512_test_epi8_mask (*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { return _mm512_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm512_setzero_si512(); } - static simdjson_inline simd8 load(const T values[64]) { - return _mm512_loadu_si512(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m512i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[64]) const { return _mm512_storeu_si512(reinterpret_cast<__m512i *>(dst), *this); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm512_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm512_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm512_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 64 - count_ones(mask) bytes of the result are significant but 64 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint64_t mask, L * output) const { - // we deliberately avoid _mm512_mask_compressstoreu_epi8 for portability - // (AMD Zen4 has terrible performance with it, it is effectively broken) - // _mm512_mask_compressstoreu_epi8 (output,~mask,*this); - __m512i compressed = _mm512_maskz_compress_epi8(~mask, *this); - _mm512_storeu_si512(output, compressed); // could use a mask - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[64]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, - int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, - int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31, - int8_t v32, int8_t v33, int8_t v34, int8_t v35, int8_t v36, int8_t v37, int8_t v38, int8_t v39, - int8_t v40, int8_t v41, int8_t v42, int8_t v43, int8_t v44, int8_t v45, int8_t v46, int8_t v47, - int8_t v48, int8_t v49, int8_t v50, int8_t v51, int8_t v52, int8_t v53, int8_t v54, int8_t v55, - int8_t v56, int8_t v57, int8_t v58, int8_t v59, int8_t v60, int8_t v61, int8_t v62, int8_t v63 - ) : simd8(_mm512_set_epi8( - v63, v62, v61, v60, v59, v58, v57, v56, - v55, v54, v53, v52, v51, v50, v49, v48, - v47, v46, v45, v44, v43, v42, v41, v40, - v39, v38, v37, v36, v35, v34, v33, v32, - v31, v30, v29, v28, v27, v26, v25, v24, - v23, v22, v21, v20, v19, v18, v17, v16, - v15, v14, v13, v12, v11, v10, v9, v8, - v7, v6, v5, v4, v3, v2, v1, v0 - )) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epi8(*this, other); } - - simdjson_inline simd8 operator>(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(*this, other),_mm512_set1_epi8(uint8_t(0x80))); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(other, *this),_mm512_set1_epi8(uint8_t(0x80))); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[64]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, - uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, - uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31, - uint8_t v32, uint8_t v33, uint8_t v34, uint8_t v35, uint8_t v36, uint8_t v37, uint8_t v38, uint8_t v39, - uint8_t v40, uint8_t v41, uint8_t v42, uint8_t v43, uint8_t v44, uint8_t v45, uint8_t v46, uint8_t v47, - uint8_t v48, uint8_t v49, uint8_t v50, uint8_t v51, uint8_t v52, uint8_t v53, uint8_t v54, uint8_t v55, - uint8_t v56, uint8_t v57, uint8_t v58, uint8_t v59, uint8_t v60, uint8_t v61, uint8_t v62, uint8_t v63 - ) : simd8(_mm512_set_epi8( - v63, v62, v61, v60, v59, v58, v57, v56, - v55, v54, v53, v52, v51, v50, v49, v48, - v47, v46, v45, v44, v43, v42, v41, v40, - v39, v38, v37, v36, v35, v34, v33, v32, - v31, v30, v29, v28, v27, v26, v25, v24, - v23, v22, v21, v20, v19, v18, v17, v16, - v15, v14, v13, v12, v11, v10, v9, v8, - v7, v6, v5, v4, v3, v2, v1, v0 - )) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm512_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm512_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epu8(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline uint64_t operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline uint64_t operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return _mm512_mask_blend_epi8(*this == uint8_t(0), _mm512_set1_epi8(0), _mm512_set1_epi8(-1)); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - - simdjson_inline bool is_ascii() const { return _mm512_movepi8_mask(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { - return !_mm512_test_epi8_mask(*this, *this); - } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return !_mm512_test_epi8_mask(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm512_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm512_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7-N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 1, "Icelake kernel should use one register per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} - simdjson_inline simd8x64(const simd8 chunk0) : chunks{chunk0} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr)} {} - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - this->chunks[0].compress(mask, output); - return 64 - count_ones(mask); - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - } - - simdjson_inline simd8 reduce_or() const { - return this->chunks[0]; - } - - simdjson_inline simd8x64 bit_or(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] | mask - ); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return this->chunks[0] == mask; - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return this->chunks[0] == other.chunks[0]; - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return this->chunks[0] <= mask; - } - }; // struct simd8x64 - -} // namespace simd - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_SIMD_H -/* end file simdjson/icelake/simd.h */ -/* including simdjson/icelake/stringparsing_defs.h: #include "simdjson/icelake/stringparsing_defs.h" */ -/* begin file simdjson/icelake/stringparsing_defs.h */ -#ifndef SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H -#define SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 64; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint64_t bs_bits; - uint64_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 15 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - // store to dest unconditionally - we can overwrite the bits we don't like later - v.store(dst); - return { - static_cast(v == '\\'), // bs_bits - static_cast(v == '"'), // quote_bits - }; -} - - -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H -/* end file simdjson/icelake/stringparsing_defs.h */ -/* including simdjson/icelake/numberparsing_defs.h: #include "simdjson/icelake/numberparsing_defs.h" */ -/* begin file simdjson/icelake/numberparsing_defs.h */ -#ifndef SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H -#define SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace numberparsing { - -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - // this actually computes *16* values so we are being wasteful. - const __m128i ascii0 = _mm_set1_epi8('0'); - const __m128i mul_1_10 = - _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); - const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); - const __m128i mul_1_10000 = - _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); - const __m128i input = _mm_sub_epi8( - _mm_loadu_si128(reinterpret_cast(chars)), ascii0); - const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); - const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); - const __m128i t3 = _mm_packus_epi32(t2, t2); - const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); - return _mm_cvtsi128_si32( - t4); // only captures the sum of the first 8 digits, drop the rest -} - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace icelake -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H -/* end file simdjson/icelake/numberparsing_defs.h */ -/* end file simdjson/icelake/begin.h */ -/* including simdjson/generic/amalgamated.h for icelake: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for icelake */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for icelake: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for icelake */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for icelake */ -/* including simdjson/generic/jsoncharutils.h for icelake: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for icelake */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for icelake */ -/* including simdjson/generic/atomparsing.h for icelake: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for icelake */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace icelake { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for icelake */ -/* including simdjson/generic/dom_parser_implementation.h for icelake: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for icelake */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace icelake -} // namespace simdjson - -namespace simdjson { -namespace icelake { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for icelake */ -/* including simdjson/generic/implementation_simdjson_result_base.h for icelake: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for icelake */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for icelake */ -/* including simdjson/generic/numberparsing.h for icelake: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for icelake */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace icelake { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for icelake */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for icelake: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for icelake */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for icelake */ -/* end file simdjson/generic/amalgamated.h for icelake */ -/* including simdjson/icelake/end.h: #include "simdjson/icelake/end.h" */ -/* begin file simdjson/icelake/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_ICELAKE -SIMDJSON_UNTARGET_REGION -#endif - -/* undefining SIMDJSON_IMPLEMENTATION from "icelake" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/icelake/end.h */ - -#endif // SIMDJSON_ICELAKE_H -/* end file simdjson/icelake.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(ppc64) -/* including simdjson/ppc64.h: #include "simdjson/ppc64.h" */ -/* begin file simdjson/ppc64.h */ -#ifndef SIMDJSON_PPC64_H -#define SIMDJSON_PPC64_H - -/* including simdjson/ppc64/begin.h: #include "simdjson/ppc64/begin.h" */ -/* begin file simdjson/ppc64/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "ppc64" */ -#define SIMDJSON_IMPLEMENTATION ppc64 -/* including simdjson/ppc64/base.h: #include "simdjson/ppc64/base.h" */ -/* begin file simdjson/ppc64/base.h */ -#ifndef SIMDJSON_PPC64_BASE_H -#define SIMDJSON_PPC64_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * Implementation for ALTIVEC (PPC64). - */ -namespace ppc64 { - -class implementation; - -namespace { -namespace simd { -template struct simd8; -template struct simd8x64; -} // namespace simd -} // unnamed namespace - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_BASE_H -/* end file simdjson/ppc64/base.h */ -/* including simdjson/ppc64/intrinsics.h: #include "simdjson/ppc64/intrinsics.h" */ -/* begin file simdjson/ppc64/intrinsics.h */ -#ifndef SIMDJSON_PPC64_INTRINSICS_H -#define SIMDJSON_PPC64_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This should be the correct header whether -// you use visual studio or other compilers. -#include - -// These are defined by altivec.h in GCC toolchain, it is safe to undef them. -#ifdef bool -#undef bool -#endif - -#ifdef vector -#undef vector -#endif - -static_assert(sizeof(__vector unsigned char) <= simdjson::SIMDJSON_PADDING, "insufficient padding for ppc64"); - -#endif // SIMDJSON_PPC64_INTRINSICS_H -/* end file simdjson/ppc64/intrinsics.h */ -/* including simdjson/ppc64/bitmanipulation.h: #include "simdjson/ppc64/bitmanipulation.h" */ -/* begin file simdjson/ppc64/bitmanipulation.h */ -#ifndef SIMDJSON_PPC64_BITMANIPULATION_H -#define SIMDJSON_PPC64_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num - 1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_inline int count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num); // Visual Studio wants two underscores -} -#else -simdjson_inline int count_ones(uint64_t input_num) { - return __builtin_popcountll(input_num); -} -#endif - - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_BITMANIPULATION_H -/* end file simdjson/ppc64/bitmanipulation.h */ -/* including simdjson/ppc64/bitmask.h: #include "simdjson/ppc64/bitmask.h" */ -/* begin file simdjson/ppc64/bitmask.h */ -#ifndef SIMDJSON_PPC64_BITMASK_H -#define SIMDJSON_PPC64_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is -// encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { - // You can use the version below, however gcc sometimes miscompiles - // vec_pmsum_be, it happens somewhere around between 8 and 9th version. - // The performance boost was not noticeable, falling back to a usual - // implementation. - // __vector unsigned long long all_ones = {~0ull, ~0ull}; - // __vector unsigned long long mask = {bitmask, 0}; - // // Clang and GCC return different values for pmsum for ull so cast it to one. - // // Generally it is not specified by ALTIVEC ISA what is returned by - // // vec_pmsum_be. - // #if defined(__LITTLE_ENDIAN__) - // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[0]); - // #else - // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[1]); - // #endif - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; - return bitmask; -} - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif -/* end file simdjson/ppc64/bitmask.h */ -/* including simdjson/ppc64/numberparsing_defs.h: #include "simdjson/ppc64/numberparsing_defs.h" */ -/* begin file simdjson/ppc64/numberparsing_defs.h */ -#ifndef SIMDJSON_PPC64_NUMBERPARSING_DEFS_H -#define SIMDJSON_PPC64_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#if defined(__linux__) -#include -#elif defined(__FreeBSD__) -#include -#endif - -namespace simdjson { -namespace ppc64 { -namespace numberparsing { - -// we don't have appropriate instructions, so let us use a scalar function -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - uint64_t val; - std::memcpy(&val, chars, sizeof(uint64_t)); -#ifdef __BIG_ENDIAN__ -#if defined(__linux__) - val = bswap_64(val); -#elif defined(__FreeBSD__) - val = bswap64(val); -#endif -#endif - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace ppc64 -} // namespace simdjson - -#ifndef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_IS_BIG_ENDIAN -#define SIMDJSON_SWAR_NUMBER_PARSING 0 -#else -#define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif -#endif - -#endif // SIMDJSON_PPC64_NUMBERPARSING_DEFS_H -/* end file simdjson/ppc64/numberparsing_defs.h */ -/* including simdjson/ppc64/simd.h: #include "simdjson/ppc64/simd.h" */ -/* begin file simdjson/ppc64/simd.h */ -#ifndef SIMDJSON_PPC64_SIMD_H -#define SIMDJSON_PPC64_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace ppc64 { -namespace { -namespace simd { - -using __m128i = __vector unsigned char; - -template struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{__m128i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i &() const { - return this->value; - } - simdjson_inline operator __m128i &() { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { - return vec_or(this->value, (__m128i)other); - } - simdjson_inline Child operator&(const Child other) const { - return vec_and(this->value, (__m128i)other); - } - simdjson_inline Child operator^(const Child other) const { - return vec_xor(this->value, (__m128i)other); - } - simdjson_inline Child bit_andnot(const Child other) const { - return vec_andc(this->value, (__m128i)other); - } - simdjson_inline Child &operator|=(const Child other) { - auto this_cast = static_cast(this); - *this_cast = *this_cast | other; - return *this_cast; - } - simdjson_inline Child &operator&=(const Child other) { - auto this_cast = static_cast(this); - *this_cast = *this_cast & other; - return *this_cast; - } - simdjson_inline Child &operator^=(const Child other) { - auto this_cast = static_cast(this); - *this_cast = *this_cast ^ other; - return *this_cast; - } -}; - -template > -struct base8 : base> { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { - return (__m128i)vec_cmpeq(lhs.value, (__m128i)rhs); - } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(simd8 prev_chunk) const { - __m128i chunk = this->value; -#ifdef __LITTLE_ENDIAN__ - chunk = (__m128i)vec_reve(this->value); - prev_chunk = (__m128i)vec_reve((__m128i)prev_chunk); -#endif - chunk = (__m128i)vec_sld((__m128i)prev_chunk, (__m128i)chunk, 16 - N); -#ifdef __LITTLE_ENDIAN__ - chunk = (__m128i)vec_reve((__m128i)chunk); -#endif - return chunk; - } -}; - -// SIMD byte mask type (returned by things like eq and gt) -template <> struct simd8 : base8 { - static simdjson_inline simd8 splat(bool _value) { - return (__m128i)vec_splats((unsigned char)(-(!!_value))); - } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) - : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) - : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { - __vector unsigned long long result; - const __m128i perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40, - 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00}; - - result = ((__vector unsigned long long)vec_vbpermq((__m128i)this->value, - (__m128i)perm_mask)); -#ifdef __LITTLE_ENDIAN__ - return static_cast(result[1]); -#else - return static_cast(result[0]); -#endif - } - simdjson_inline bool any() const { - return !vec_all_eq(this->value, (__m128i)vec_splats(0)); - } - simdjson_inline simd8 operator~() const { - return this->value ^ (__m128i)splat(true); - } -}; - -template struct base8_numeric : base8 { - static simdjson_inline simd8 splat(T value) { - (void)value; - return (__m128i)vec_splats(value); - } - static simdjson_inline simd8 zero() { return splat(0); } - static simdjson_inline simd8 load(const T values[16]) { - return (__m128i)(vec_vsx_ld(0, reinterpret_cast(values))); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, - T v5, T v6, T v7, T v8, T v9, - T v10, T v11, T v12, T v13, - T v14, T v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, - v14, v15); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) - : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { - vec_vsx_st(this->value, 0, reinterpret_cast<__m128i *>(dst)); - } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { - return (__m128i)((__m128i)this->value + (__m128i)other); - } - simdjson_inline simd8 operator-(const simd8 other) const { - return (__m128i)((__m128i)this->value - (__m128i)other); - } - simdjson_inline simd8 &operator+=(const simd8 other) { - *this = *this + other; - return *static_cast *>(this); - } - simdjson_inline simd8 &operator-=(const simd8 other) { - *this = *this - other; - return *static_cast *>(this); - } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior - // for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return (__m128i)vec_perm((__m128i)lookup_table, (__m128i)lookup_table, this->value); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted - // as a bitset). Passing a 0 value for mask would be equivalent to writing out - // every byte to output. Only the first 16 - count_ones(mask) bytes of the - // result are significant but 16 bytes get written. Design consideration: it - // seems like a function with the signature simd8 compress(uint32_t mask) - // would be sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L *output) const { - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - using internal::thintable_epi8; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. -#ifdef __LITTLE_ENDIAN__ - __m128i shufmask = (__m128i)(__vector unsigned long long){ - thintable_epi8[mask1], thintable_epi8[mask2]}; -#else - __m128i shufmask = (__m128i)(__vector unsigned long long){ - thintable_epi8[mask2], thintable_epi8[mask1]}; - shufmask = (__m128i)vec_reve((__m128i)shufmask); -#endif - // we increment by 0x08 the second half of the mask - shufmask = ((__m128i)shufmask) + - ((__m128i)(__vector int){0, 0, 0x08080808, 0x08080808}); - - // this is the version "nearly pruned" - __m128i pruned = vec_perm(this->value, this->value, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - __m128i compactmask = - vec_vsx_ld(0, reinterpret_cast(pshufb_combine_table + pop1 * 8)); - __m128i answer = vec_perm(pruned, (__m128i)vec_splats(0), compactmask); - vec_vsx_st(answer, 0, reinterpret_cast<__m128i *>(output)); - } - - template - simdjson_inline simd8 - lookup_16(L replace0, L replace1, L replace2, L replace3, L replace4, - L replace5, L replace6, L replace7, L replace8, L replace9, - L replace10, L replace11, L replace12, L replace13, L replace14, - L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, replace4, replace5, replace6, - replace7, replace8, replace9, replace10, replace11, replace12, - replace13, replace14, replace15)); - } -}; - -// Signed bytes -template <> struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) - : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t *values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3, - int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, - int8_t v12, int8_t v13, int8_t v14, int8_t v15) - : simd8((__m128i)(__vector signed char){v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, - v15}) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 - repeat_16(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, - int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, - int8_t v12, int8_t v13, int8_t v14, int8_t v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15); - } - - // Order-sensitive comparisons - simdjson_inline simd8 - max_val(const simd8 other) const { - return (__m128i)vec_max((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - simdjson_inline simd8 - min_val(const simd8 other) const { - return (__m128i)vec_min((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - simdjson_inline simd8 - operator>(const simd8 other) const { - return (__m128i)vec_cmpgt((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - simdjson_inline simd8 - operator<(const simd8 other) const { - return (__m128i)vec_cmplt((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } -}; - -// Unsigned bytes -template <> struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) - : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t *values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline - simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, - uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, - uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) - : simd8((__m128i){v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15}) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 - repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, - uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, - uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, - uint8_t v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15); - } - - // Saturated math - simdjson_inline simd8 - saturating_add(const simd8 other) const { - return (__m128i)vec_adds(this->value, (__m128i)other); - } - simdjson_inline simd8 - saturating_sub(const simd8 other) const { - return (__m128i)vec_subs(this->value, (__m128i)other); - } - - // Order-specific operations - simdjson_inline simd8 - max_val(const simd8 other) const { - return (__m128i)vec_max(this->value, (__m128i)other); - } - simdjson_inline simd8 - min_val(const simd8 other) const { - return (__m128i)vec_min(this->value, (__m128i)other); - } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 - gt_bits(const simd8 other) const { - return this->saturating_sub(other); - } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 - lt_bits(const simd8 other) const { - return other.saturating_sub(*this); - } - simdjson_inline simd8 - operator<=(const simd8 other) const { - return other.max_val(*this) == other; - } - simdjson_inline simd8 - operator>=(const simd8 other) const { - return other.min_val(*this) == other; - } - simdjson_inline simd8 - operator>(const simd8 other) const { - return this->gt_bits(other).any_bits_set(); - } - simdjson_inline simd8 - operator<(const simd8 other) const { - return this->gt_bits(other).any_bits_set(); - } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { - return (__m128i)vec_cmpeq(this->value, (__m128i)vec_splats(uint8_t(0))); - } - simdjson_inline simd8 bits_not_set(simd8 bits) const { - return (*this & bits).bits_not_set(); - } - simdjson_inline simd8 any_bits_set() const { - return ~this->bits_not_set(); - } - simdjson_inline simd8 any_bits_set(simd8 bits) const { - return ~this->bits_not_set(bits); - } - simdjson_inline bool bits_not_set_anywhere() const { - return vec_all_eq(this->value, (__m128i)vec_splats(0)); - } - simdjson_inline bool any_bits_set_anywhere() const { - return !bits_not_set_anywhere(); - } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { - return vec_all_eq(vec_and(this->value, (__m128i)bits), - (__m128i)vec_splats(0)); - } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { - return !bits_not_set_anywhere(bits); - } - template simdjson_inline simd8 shr() const { - return simd8( - (__m128i)vec_sr(this->value, (__m128i)vec_splat_u8(N))); - } - template simdjson_inline simd8 shl() const { - return simd8( - (__m128i)vec_sl(this->value, (__m128i)vec_splat_u8(N))); - } -}; - -template struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, - "PPC64 kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64 &o) = delete; // no copy allowed - simd8x64 & - operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, - const simd8 chunk2, const simd8 chunk3) - : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_inline simd8x64(const T ptr[64]) - : chunks{simd8::load(ptr), simd8::load(ptr + 16), - simd8::load(ptr + 32), simd8::load(ptr + 48)} {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr + sizeof(simd8) * 0); - this->chunks[1].store(ptr + sizeof(simd8) * 1); - this->chunks[2].store(ptr + sizeof(simd8) * 2); - this->chunks[3].store(ptr + sizeof(simd8) * 3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | - (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t compress(uint64_t mask, T *output) const { - this->chunks[0].compress(uint16_t(mask), output); - this->chunks[1].compress(uint16_t(mask >> 16), - output + 16 - count_ones(mask & 0xFFFF)); - this->chunks[2].compress(uint16_t(mask >> 32), - output + 32 - count_ones(mask & 0xFFFFFFFF)); - this->chunks[3].compress(uint16_t(mask >> 48), - output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); - return 64 - count_ones(mask); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r1 = this->chunks[1].to_bitmask(); - uint64_t r2 = this->chunks[2].to_bitmask(); - uint64_t r3 = this->chunks[3].to_bitmask(); - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] == mask, this->chunks[1] == mask, - this->chunks[2] == mask, this->chunks[3] == mask) - .to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64(this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3]) - .to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] <= mask, this->chunks[1] <= mask, - this->chunks[2] <= mask, this->chunks[3] <= mask) - .to_bitmask(); - } -}; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_SIMD_INPUT_H -/* end file simdjson/ppc64/simd.h */ -/* including simdjson/ppc64/stringparsing_defs.h: #include "simdjson/ppc64/stringparsing_defs.h" */ -/* begin file simdjson/ppc64/stringparsing_defs.h */ -#ifndef SIMDJSON_PPC64_STRINGPARSING_DEFS_H -#define SIMDJSON_PPC64_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/simd.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote - copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { - return ((bs_bits - 1) & quote_bits) != 0; - } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { - return trailing_zeroes(quote_bits); - } - simdjson_inline int backslash_index() { - return trailing_zeroes(bs_bits); - } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote -backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), - "backslash and quote finder must process fewer than " - "SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + sizeof(v0)); - v0.store(dst); - v1.store(dst + sizeof(v0)); - - // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on - // PPC; therefore, we smash them together into a 64-byte mask and get the - // bitmask from there. - uint64_t bs_and_quote = - simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; -} - - -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_STRINGPARSING_DEFS_H -/* end file simdjson/ppc64/stringparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 -/* end file simdjson/ppc64/begin.h */ -/* including simdjson/generic/amalgamated.h for ppc64: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for ppc64 */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for ppc64: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for ppc64 */ -/* including simdjson/generic/jsoncharutils.h for ppc64: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for ppc64 */ -/* including simdjson/generic/atomparsing.h for ppc64: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace ppc64 { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for ppc64 */ -/* including simdjson/generic/dom_parser_implementation.h for ppc64: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace ppc64 -} // namespace simdjson - -namespace simdjson { -namespace ppc64 { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for ppc64 */ -/* including simdjson/generic/implementation_simdjson_result_base.h for ppc64: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for ppc64 */ -/* including simdjson/generic/numberparsing.h for ppc64: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace ppc64 { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for ppc64 */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for ppc64: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for ppc64 */ -/* end file simdjson/generic/amalgamated.h for ppc64 */ -/* including simdjson/ppc64/end.h: #include "simdjson/ppc64/end.h" */ -/* begin file simdjson/ppc64/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT -/* undefining SIMDJSON_IMPLEMENTATION from "ppc64" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/ppc64/end.h */ - -#endif // SIMDJSON_PPC64_H -/* end file simdjson/ppc64.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(westmere) -/* including simdjson/westmere.h: #include "simdjson/westmere.h" */ -/* begin file simdjson/westmere.h */ -#ifndef SIMDJSON_WESTMERE_H -#define SIMDJSON_WESTMERE_H - -/* including simdjson/westmere/begin.h: #include "simdjson/westmere/begin.h" */ -/* begin file simdjson/westmere/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "westmere" */ -#define SIMDJSON_IMPLEMENTATION westmere -/* including simdjson/westmere/base.h: #include "simdjson/westmere/base.h" */ -/* begin file simdjson/westmere/base.h */ -#ifndef SIMDJSON_WESTMERE_BASE_H -#define SIMDJSON_WESTMERE_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE -namespace simdjson { -/** - * Implementation for Westmere (Intel SSE4.2). - */ -namespace westmere { - -class implementation; - -namespace { -namespace simd { - -template struct simd8; -template struct simd8x64; - -} // namespace simd -} // unnamed namespace - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BASE_H -/* end file simdjson/westmere/base.h */ -/* including simdjson/westmere/intrinsics.h: #include "simdjson/westmere/intrinsics.h" */ -/* begin file simdjson/westmere/intrinsics.h */ -#ifndef SIMDJSON_WESTMERE_INTRINSICS_H -#define SIMDJSON_WESTMERE_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - */ -#include // for _mm_alignr_epi8 -#include // for _mm_clmulepi64_si128 -#endif - -static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); - -#endif // SIMDJSON_WESTMERE_INTRINSICS_H -/* end file simdjson/westmere/intrinsics.h */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_WESTMERE -SIMDJSON_TARGET_REGION("sse4.2,pclmul,popcnt") -#endif - -/* including simdjson/westmere/bitmanipulation.h: #include "simdjson/westmere/bitmanipulation.h" */ -/* begin file simdjson/westmere/bitmanipulation.h */ -#ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H -#define SIMDJSON_WESTMERE_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO -} - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num);// Visual Studio wants two underscores -} -#else -simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); -} -#endif - - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BITMANIPULATION_H -/* end file simdjson/westmere/bitmanipulation.h */ -/* including simdjson/westmere/bitmask.h: #include "simdjson/westmere/bitmask.h" */ -/* begin file simdjson/westmere/bitmask.h */ -#ifndef SIMDJSON_WESTMERE_BITMASK_H -#define SIMDJSON_WESTMERE_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { - // There should be no such thing with a processing supporting avx2 - // but not clmul. - __m128i all_ones = _mm_set1_epi8('\xFF'); - __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); - return _mm_cvtsi128_si64(result); -} - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BITMASK_H -/* end file simdjson/westmere/bitmask.h */ -/* including simdjson/westmere/numberparsing_defs.h: #include "simdjson/westmere/numberparsing_defs.h" */ -/* begin file simdjson/westmere/numberparsing_defs.h */ -#ifndef SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H -#define SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H - -/* including simdjson/westmere/base.h: #include "simdjson/westmere/base.h" */ -/* begin file simdjson/westmere/base.h */ -#ifndef SIMDJSON_WESTMERE_BASE_H -#define SIMDJSON_WESTMERE_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE -namespace simdjson { -/** - * Implementation for Westmere (Intel SSE4.2). - */ -namespace westmere { - -class implementation; - -namespace { -namespace simd { - -template struct simd8; -template struct simd8x64; - -} // namespace simd -} // unnamed namespace - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BASE_H -/* end file simdjson/westmere/base.h */ -/* including simdjson/westmere/intrinsics.h: #include "simdjson/westmere/intrinsics.h" */ -/* begin file simdjson/westmere/intrinsics.h */ -#ifndef SIMDJSON_WESTMERE_INTRINSICS_H -#define SIMDJSON_WESTMERE_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - */ -#include // for _mm_alignr_epi8 -#include // for _mm_clmulepi64_si128 -#endif - -static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); - -#endif // SIMDJSON_WESTMERE_INTRINSICS_H -/* end file simdjson/westmere/intrinsics.h */ - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace numberparsing { - -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - // this actually computes *16* values so we are being wasteful. - const __m128i ascii0 = _mm_set1_epi8('0'); - const __m128i mul_1_10 = - _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); - const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); - const __m128i mul_1_10000 = - _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); - const __m128i input = _mm_sub_epi8( - _mm_loadu_si128(reinterpret_cast(chars)), ascii0); - const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); - const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); - const __m128i t3 = _mm_packus_epi32(t2, t2); - const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); - return _mm_cvtsi128_si32( - t4); // only captures the sum of the first 8 digits, drop the rest -} - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#if SIMDJSON_IS_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // SIMDJSON_IS_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; -} - -} // namespace numberparsing -} // namespace westmere -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H -/* end file simdjson/westmere/numberparsing_defs.h */ -/* including simdjson/westmere/simd.h: #include "simdjson/westmere/simd.h" */ -/* begin file simdjson/westmere/simd.h */ -#ifndef SIMDJSON_WESTMERE_SIMD_H -#define SIMDJSON_WESTMERE_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace simd { - - template - struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{__m128i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i&() const { return this->value; } - simdjson_inline operator __m128i&() { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - template> - struct base8: base> { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return _mm_alignr_epi8(*this, prev_chunk, 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } - simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } - static simdjson_inline simd8 load(const T values[16]) { - return _mm_loadu_si128(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - __m128i shufmask = _mm_set_epi64x(thintable_epi8[mask2], thintable_epi8[mask1]); - // we increment by 0x08 the second half of the mask - shufmask = - _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); - // this is the version "nearly pruned" - __m128i pruned = _mm_shuffle_epi8(*this, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - __m128i compactmask = - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8)); - __m128i answer = _mm_shuffle_epi8(pruned, compactmask); - _mm_storeu_si128(reinterpret_cast<__m128i *>(output), answer); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - this->chunks[2].store(ptr+sizeof(simd8)*2); - this->chunks[3].store(ptr+sizeof(simd8)*3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - this->chunks[0].compress(uint16_t(mask), output); - this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); - this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); - this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); - return 64 - count_ones(mask); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask() ); - uint64_t r1 = this->chunks[1].to_bitmask() ; - uint64_t r2 = this->chunks[2].to_bitmask() ; - uint64_t r3 = this->chunks[3].to_bitmask() ; - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_SIMD_INPUT_H -/* end file simdjson/westmere/simd.h */ -/* including simdjson/westmere/stringparsing_defs.h: #include "simdjson/westmere/stringparsing_defs.h" */ -/* begin file simdjson/westmere/stringparsing_defs.h */ -#ifndef SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H -#define SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H - -/* including simdjson/westmere/bitmanipulation.h: #include "simdjson/westmere/bitmanipulation.h" */ -/* begin file simdjson/westmere/bitmanipulation.h */ -#ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H -#define SIMDJSON_WESTMERE_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO -} - -#if SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num);// Visual Studio wants two underscores -} -#else -simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); -} -#endif - - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BITMANIPULATION_H -/* end file simdjson/westmere/bitmanipulation.h */ -/* including simdjson/westmere/simd.h: #include "simdjson/westmere/simd.h" */ -/* begin file simdjson/westmere/simd.h */ -#ifndef SIMDJSON_WESTMERE_SIMD_H -#define SIMDJSON_WESTMERE_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace simd { - - template - struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{__m128i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i&() const { return this->value; } - simdjson_inline operator __m128i&() { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - template> - struct base8: base> { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return _mm_alignr_epi8(*this, prev_chunk, 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } - simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } - static simdjson_inline simd8 load(const T values[16]) { - return _mm_loadu_si128(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - __m128i shufmask = _mm_set_epi64x(thintable_epi8[mask2], thintable_epi8[mask1]); - // we increment by 0x08 the second half of the mask - shufmask = - _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); - // this is the version "nearly pruned" - __m128i pruned = _mm_shuffle_epi8(*this, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - __m128i compactmask = - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8)); - __m128i answer = _mm_shuffle_epi8(pruned, compactmask); - _mm_storeu_si128(reinterpret_cast<__m128i *>(output), answer); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - this->chunks[2].store(ptr+sizeof(simd8)*2); - this->chunks[3].store(ptr+sizeof(simd8)*3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - this->chunks[0].compress(uint16_t(mask), output); - this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); - this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); - this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); - return 64 - count_ones(mask); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask() ); - uint64_t r1 = this->chunks[1].to_bitmask() ; - uint64_t r2 = this->chunks[2].to_bitmask() ; - uint64_t r3 = this->chunks[3].to_bitmask() ; - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_SIMD_INPUT_H -/* end file simdjson/westmere/simd.h */ - -namespace simdjson { -namespace westmere { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + 16); - v0.store(dst); - v1.store(dst + 16); - uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; -} - - -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H -/* end file simdjson/westmere/stringparsing_defs.h */ -/* end file simdjson/westmere/begin.h */ -/* including simdjson/generic/amalgamated.h for westmere: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for westmere */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for westmere: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for westmere */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for westmere */ -/* including simdjson/generic/jsoncharutils.h for westmere: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for westmere */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for westmere */ -/* including simdjson/generic/atomparsing.h for westmere: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for westmere */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace westmere { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for westmere */ -/* including simdjson/generic/dom_parser_implementation.h for westmere: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for westmere */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace westmere -} // namespace simdjson - -namespace simdjson { -namespace westmere { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for westmere */ -/* including simdjson/generic/implementation_simdjson_result_base.h for westmere: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for westmere */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for westmere */ -/* including simdjson/generic/numberparsing.h for westmere: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for westmere */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace westmere { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for westmere */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for westmere: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for westmere */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for westmere */ -/* end file simdjson/generic/amalgamated.h for westmere */ -/* including simdjson/westmere/end.h: #include "simdjson/westmere/end.h" */ -/* begin file simdjson/westmere/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_WESTMERE -SIMDJSON_UNTARGET_REGION -#endif - -/* undefining SIMDJSON_IMPLEMENTATION from "westmere" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/westmere/end.h */ - -#endif // SIMDJSON_WESTMERE_H -/* end file simdjson/westmere.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(lasx) -/* including simdjson/lasx.h: #include "simdjson/lasx.h" */ -/* begin file simdjson/lasx.h */ -#ifndef SIMDJSON_LASX_H -#define SIMDJSON_LASX_H - -/* including simdjson/lasx/begin.h: #include "simdjson/lasx/begin.h" */ -/* begin file simdjson/lasx/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "lasx" */ -#define SIMDJSON_IMPLEMENTATION lasx -#include // This is a hack. We should not need to put this include here. -#if SIMDJSON_CAN_ALWAYS_RUN_LASX -// nothing needed. -#else -SIMDJSON_TARGET_REGION("lasx,lsx") -#endif - -/* including simdjson/lasx/base.h: #include "simdjson/lasx/base.h" */ -/* begin file simdjson/lasx/base.h */ -#ifndef SIMDJSON_LASX_BASE_H -#define SIMDJSON_LASX_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * Implementation for LASX. - */ -namespace lasx { - -class implementation; - -namespace { -namespace simd { -template struct simd8; -template struct simd8x64; -} // namespace simd -} // unnamed namespace - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_LASX_BASE_H -/* end file simdjson/lasx/base.h */ -/* including simdjson/lasx/intrinsics.h: #include "simdjson/lasx/intrinsics.h" */ -/* begin file simdjson/lasx/intrinsics.h */ -#ifndef SIMDJSON_LASX_INTRINSICS_H -#define SIMDJSON_LASX_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include - -static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for LoongArch ASX"); - -#endif // SIMDJSON_LASX_INTRINSICS_H -/* end file simdjson/lasx/intrinsics.h */ -/* including simdjson/lasx/bitmanipulation.h: #include "simdjson/lasx/bitmanipulation.h" */ -/* begin file simdjson/lasx/bitmanipulation.h */ -#ifndef SIMDJSON_LASX_BITMANIPULATION_H -#define SIMDJSON_LASX_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmask.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { - return __builtin_ctzll(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { - return __builtin_clzll(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int count_ones(uint64_t input_num) { - return __lasx_xvpickve2gr_w(__lasx_xvpcnt_d(__m256i(v4u64{input_num, 0, 0, 0})), 0); -} - - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_LASX_BITMANIPULATION_H -/* end file simdjson/lasx/bitmanipulation.h */ -/* including simdjson/lasx/bitmask.h: #include "simdjson/lasx/bitmask.h" */ -/* begin file simdjson/lasx/bitmask.h */ -#ifndef SIMDJSON_LASX_BITMASK_H -#define SIMDJSON_LASX_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; - return bitmask; -} - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif -/* end file simdjson/lasx/bitmask.h */ -/* including simdjson/lasx/numberparsing_defs.h: #include "simdjson/lasx/numberparsing_defs.h" */ -/* begin file simdjson/lasx/numberparsing_defs.h */ -#ifndef SIMDJSON_LASX_NUMBERPARSING_DEFS_H -#define SIMDJSON_LASX_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace lasx { -namespace numberparsing { - -// we don't have appropriate instructions, so let us use a scalar function -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - uint64_t val; - std::memcpy(&val, chars, sizeof(uint64_t)); - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); - return answer; -} - -} // namespace numberparsing -} // namespace lasx -} // namespace simdjson - -#ifndef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_IS_BIG_ENDIAN -#define SIMDJSON_SWAR_NUMBER_PARSING 0 -#else -#define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif -#endif - -#endif // SIMDJSON_LASX_NUMBERPARSING_DEFS_H -/* end file simdjson/lasx/numberparsing_defs.h */ -/* including simdjson/lasx/simd.h: #include "simdjson/lasx/simd.h" */ -/* begin file simdjson/lasx/simd.h */ -#ifndef SIMDJSON_LASX_SIMD_H -#define SIMDJSON_LASX_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m256i value; - - // Zero constructor - simdjson_inline base() : value{__m256i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m256i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m256i&() const { return this->value; } - simdjson_inline operator __m256i&() { return this->value; } - simdjson_inline operator const v32i8&() const { return (v32i8&)this->value; } - simdjson_inline operator v32i8&() { return (v32i8&)this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return __lasx_xvor_v(*this, other); } - simdjson_inline Child operator&(const Child other) const { return __lasx_xvand_v(*this, other); } - simdjson_inline Child operator^(const Child other) const { return __lasx_xvxor_v(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return __lasx_xvandn_v(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8: base> { - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m256i _value) : base>(_value) {} - - friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return __lasx_xvseq_b(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - __m256i hi = __lasx_xvbsll_v(*this, N); - __m256i lo = __lasx_xvbsrl_v(*this, 16 - N); - __m256i tmp = __lasx_xvbsrl_v(prev_chunk, 16 - N); - lo = __lasx_xvpermi_q(lo, tmp, 0x21); - return __lasx_xvor_v(hi, lo); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { return __lasx_xvreplgr2vr_b(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m256i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { - __m256i mask = __lasx_xvmskltz_b(*this); - return (__lasx_xvpickve2gr_w(mask, 4) << 16) | (__lasx_xvpickve2gr_w(mask, 0)); - } - simdjson_inline bool any() const { - __m256i v = __lasx_xvmsknz_b(*this); - return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); - } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { - return __lasx_xvreplgr2vr_b(_value); - } - static simdjson_inline simd8 zero() { return __lasx_xvldi(0); } - static simdjson_inline simd8 load(const T values[32]) { - return __lasx_xvld(reinterpret_cast(values), 0); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m256i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[32]) const { - return __lasx_xvst(*this, reinterpret_cast<__m256i *>(dst), 0); - } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return __lasx_xvadd_b(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return __lasx_xvsub_b(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return __lasx_xvshuf_b(lookup_table, lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - template - simdjson_inline void compress(uint32_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by haswell - // lasx do it in 4 steps, first 8 bytes and then second 8 bytes... - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // second significant 8 bits - uint8_t mask3 = uint8_t(mask >> 16); // ... - uint8_t mask4 = uint8_t(mask >> 24); // ... - // next line just loads the 64-bit values thintable_epi8[mask{1,2,3,4}] - // into a 256-bit register. - __m256i shufmask = {int64_t(thintable_epi8[mask1]), int64_t(thintable_epi8[mask2]) + 0x0808080808080808, int64_t(thintable_epi8[mask3]), int64_t(thintable_epi8[mask4]) + 0x0808080808080808}; - // this is the version "nearly pruned" - __m256i pruned = __lasx_xvshuf_b(*this, *this, shufmask); - // we still need to put the pieces back together. - // we compute the popcount of the first words: - int pop1 = BitsSetTable256mul2[mask1]; - int pop2 = BitsSetTable256mul2[mask2]; - int pop3 = BitsSetTable256mul2[mask3]; - - // then load the corresponding mask - __m256i masklo = __lasx_xvldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop1 * 8); - __m256i maskhi = __lasx_xvldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop3 * 8); - __m256i compactmask = __lasx_xvpermi_q(maskhi, masklo, 0x20); - __m256i answer = __lasx_xvshuf_b(pruned, pruned, compactmask); - __lasx_xvst(answer, reinterpret_cast(output), 0); - uint64_t value3 = __lasx_xvpickve2gr_du(answer, 2); - uint64_t value4 = __lasx_xvpickve2gr_du(answer, 3); - uint64_t *pos = reinterpret_cast(reinterpret_cast(output) + 16 - (pop1 + pop2) / 2); - pos[0] = value3; - pos[1] = value4; - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, - int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, - int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31 - ) : simd8({ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v16,v17,v18,v19,v20,v21,v22,v23, - v24,v25,v26,v27,v28,v29,v30,v31 - }) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return __lasx_xvmax_b(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return __lasx_xvmin_b(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return __lasx_xvslt_b(other, *this); } - simdjson_inline simd8 operator<(const simd8 other) const { return __lasx_xvslt_b(*this, other); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, - uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, - uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31 - ) : simd8(__m256i(v32u8{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v16,v17,v18,v19,v20,v21,v22,v23, - v24,v25,v26,v27,v28,v29,v30,v31 - })) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return __lasx_xvsadd_bu(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return __lasx_xvssub_bu(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return __lasx_xvmax_bu(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return __lasx_xvmin_bu(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { - __m256i mask = __lasx_xvmskltz_b(*this); - return (0 == __lasx_xvpickve2gr_w(mask, 0)) && (0 == __lasx_xvpickve2gr_w(mask, 4)); - } - simdjson_inline bool bits_not_set_anywhere() const { - __m256i v = __lasx_xvmsknz_b(*this); - return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); - } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { - __m256i v = __lasx_xvmsknz_b(__lasx_xvand_v(*this, bits)); - return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); - } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(__lasx_xvsrli_b(*this, N)); } - template - simdjson_inline simd8 shl() const { return simd8(__lasx_xvslli_b(*this, N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 2, "LASX kernel should use two registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+32)} {} - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - uint32_t mask1 = uint32_t(mask); - uint32_t mask2 = uint32_t(mask >> 32); - __m256i zcnt = __lasx_xvpcnt_w(__m256i(v4u64{~mask, 0, 0, 0})); - uint64_t zcnt1 = __lasx_xvpickve2gr_wu(zcnt, 0); - uint64_t zcnt2 = __lasx_xvpickve2gr_wu(zcnt, 1); - // There should be a critical value which processes in scaler is faster. - if (zcnt1) - this->chunks[0].compress(mask1, output); - if (zcnt2) - this->chunks[1].compress(mask2, output + zcnt1); - return zcnt1 + zcnt2; - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - } - - simdjson_inline uint64_t to_bitmask() const { - __m256i mask0 = __lasx_xvmskltz_b(this->chunks[0]); - __m256i mask1 = __lasx_xvmskltz_b(this->chunks[1]); - __m256i mask_tmp = __lasx_xvpickve_w(mask0, 4); - __m256i tmp = __lasx_xvpickve_w(mask1, 4); - mask0 = __lasx_xvinsve0_w(mask0, mask1, 1); - mask_tmp = __lasx_xvinsve0_w(mask_tmp, tmp, 1); - return __lasx_xvpickve2gr_du(__lasx_xvpackev_h(mask_tmp, mask0), 0); - } - - simdjson_inline simd8 reduce_or() const { - return this->chunks[0] | this->chunks[1]; - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_LASX_SIMD_H -/* end file simdjson/lasx/simd.h */ -/* including simdjson/lasx/stringparsing_defs.h: #include "simdjson/lasx/stringparsing_defs.h" */ -/* begin file simdjson/lasx/stringparsing_defs.h */ -#ifndef SIMDJSON_LASX_STRINGPARSING_DEFS_H -#define SIMDJSON_LASX_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - v.store(dst); - return { - static_cast((v == '\\').to_bitmask()), // bs_bits - static_cast((v == '"').to_bitmask()), // quote_bits - }; -} - - -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_LASX_STRINGPARSING_DEFS_H -/* end file simdjson/lasx/stringparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 - - -/* end file simdjson/lasx/begin.h */ -/* including simdjson/generic/amalgamated.h for lasx: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for lasx */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for lasx: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for lasx */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for lasx */ -/* including simdjson/generic/jsoncharutils.h for lasx: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for lasx */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for lasx */ -/* including simdjson/generic/atomparsing.h for lasx: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for lasx */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace lasx { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for lasx */ -/* including simdjson/generic/dom_parser_implementation.h for lasx: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for lasx */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace lasx -} // namespace simdjson - -namespace simdjson { -namespace lasx { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for lasx */ -/* including simdjson/generic/implementation_simdjson_result_base.h for lasx: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for lasx */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for lasx */ -/* including simdjson/generic/numberparsing.h for lasx: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for lasx */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace lasx { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for lasx */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for lasx: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for lasx */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for lasx */ -/* end file simdjson/generic/amalgamated.h for lasx */ -/* including simdjson/lasx/end.h: #include "simdjson/lasx/end.h" */ -/* begin file simdjson/lasx/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT -/* undefining SIMDJSON_IMPLEMENTATION from "lasx" */ -#undef SIMDJSON_IMPLEMENTATION - - -#if SIMDJSON_CAN_ALWAYS_RUN_LASX -// nothing needed. -#else -SIMDJSON_UNTARGET_REGION -#endif -/* end file simdjson/lasx/end.h */ - -#endif // SIMDJSON_LASX_H -/* end file simdjson/lasx.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(lsx) -/* including simdjson/lsx.h: #include "simdjson/lsx.h" */ -/* begin file simdjson/lsx.h */ -#ifndef SIMDJSON_LSX_H -#define SIMDJSON_LSX_H - -/* including simdjson/lsx/begin.h: #include "simdjson/lsx/begin.h" */ -/* begin file simdjson/lsx/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "lsx" */ -#define SIMDJSON_IMPLEMENTATION lsx -/* including simdjson/lsx/base.h: #include "simdjson/lsx/base.h" */ -/* begin file simdjson/lsx/base.h */ -#ifndef SIMDJSON_LSX_BASE_H -#define SIMDJSON_LSX_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * Implementation for LSX. - */ -namespace lsx { - -class implementation; - -namespace { -namespace simd { -template struct simd8; -template struct simd8x64; -} // namespace simd -} // unnamed namespace - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_LSX_BASE_H -/* end file simdjson/lsx/base.h */ -/* including simdjson/lsx/intrinsics.h: #include "simdjson/lsx/intrinsics.h" */ -/* begin file simdjson/lsx/intrinsics.h */ -#ifndef SIMDJSON_LSX_INTRINSICS_H -#define SIMDJSON_LSX_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for LoongArch SX"); - -#endif // SIMDJSON_LSX_INTRINSICS_H -/* end file simdjson/lsx/intrinsics.h */ -/* including simdjson/lsx/bitmanipulation.h: #include "simdjson/lsx/bitmanipulation.h" */ -/* begin file simdjson/lsx/bitmanipulation.h */ -#ifndef SIMDJSON_LSX_BITMANIPULATION_H -#define SIMDJSON_LSX_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmask.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { - return __builtin_ctzll(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { - return __builtin_clzll(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int count_ones(uint64_t input_num) { - return __lsx_vpickve2gr_w(__lsx_vpcnt_d(__m128i(v2u64{input_num, 0})), 0); -} - - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_LSX_BITMANIPULATION_H -/* end file simdjson/lsx/bitmanipulation.h */ -/* including simdjson/lsx/bitmask.h: #include "simdjson/lsx/bitmask.h" */ -/* begin file simdjson/lsx/bitmask.h */ -#ifndef SIMDJSON_LSX_BITMASK_H -#define SIMDJSON_LSX_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; - return bitmask; -} - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif -/* end file simdjson/lsx/bitmask.h */ -/* including simdjson/lsx/numberparsing_defs.h: #include "simdjson/lsx/numberparsing_defs.h" */ -/* begin file simdjson/lsx/numberparsing_defs.h */ -#ifndef SIMDJSON_LSX_NUMBERPARSING_DEFS_H -#define SIMDJSON_LSX_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace lsx { -namespace numberparsing { - -// we don't have appropriate instructions, so let us use a scalar function -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - uint64_t val; - std::memcpy(&val, chars, sizeof(uint64_t)); - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); - return answer; -} - -} // namespace numberparsing -} // namespace lsx -} // namespace simdjson - -#ifndef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_IS_BIG_ENDIAN -#define SIMDJSON_SWAR_NUMBER_PARSING 0 -#else -#define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif -#endif - -#endif // SIMDJSON_LSX_NUMBERPARSING_DEFS_H -/* end file simdjson/lsx/numberparsing_defs.h */ -/* including simdjson/lsx/simd.h: #include "simdjson/lsx/simd.h" */ -/* begin file simdjson/lsx/simd.h */ -#ifndef SIMDJSON_LSX_SIMD_H -#define SIMDJSON_LSX_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{__m128i()} {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i&() const { return this->value; } - simdjson_inline operator __m128i&() { return this->value; } - simdjson_inline operator const v16i8&() const { return (v16i8&)this->value; } - simdjson_inline operator v16i8&() { return (v16i8&)this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return __lsx_vor_v(*this, other); } - simdjson_inline Child operator&(const Child other) const { return __lsx_vand_v(*this, other); } - simdjson_inline Child operator^(const Child other) const { return __lsx_vxor_v(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return __lsx_vandn_v(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8: base> { - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return __lsx_vseq_b(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return __lsx_vor_v(__lsx_vbsll_v(*this, N), __lsx_vbsrl_v(prev_chunk, 16 - N)); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8: base8 { - static simdjson_inline simd8 splat(bool _value) { - return __lsx_vreplgr2vr_b(uint8_t(-(!!_value))); - } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return __lsx_vpickve2gr_w(__lsx_vmskltz_b(*this), 0); } - simdjson_inline bool any() const { return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(*this), 0); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric: base8 { - static simdjson_inline simd8 splat(T _value) { return __lsx_vreplgr2vr_b(_value); } - static simdjson_inline simd8 zero() { return __lsx_vldi(0); } - static simdjson_inline simd8 load(const T values[16]) { - return __lsx_vld(reinterpret_cast(values), 0); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { - return __lsx_vst(*this, reinterpret_cast<__m128i *>(dst), 0); - } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return __lsx_vadd_b(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return __lsx_vsub_b(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return __lsx_vshuf_b(lookup_table, lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - template - simdjson_inline void compress(uint16_t mask, L * output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by haswell - // lsx do it in 2 steps, first 8 bytes and then second 8 bytes... - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register. - __m128i shufmask = {int64_t(thintable_epi8[mask1]), int64_t(thintable_epi8[mask2]) + 0x0808080808080808}; - // this is the version "nearly pruned" - __m128i pruned = __lsx_vshuf_b(*this, *this, shufmask); - // we still need to put the pieces back together. - // we compute the popcount of the first words: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask - __m128i compactmask = __lsx_vldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop1 * 8); - __m128i answer = __lsx_vshuf_b(pruned, pruned, compactmask); - __lsx_vst(answer, reinterpret_cast(output), 0); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[16]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8({ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return __lsx_vmax_b(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return __lsx_vmin_b(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return __lsx_vslt_b(other, *this); } - simdjson_inline simd8 operator<(const simd8 other) const { return __lsx_vslt_b(*this, other); } - }; - - // Unsigned bytes - template<> - struct simd8: base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(__m128i(v16u8{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - })) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return __lsx_vsadd_bu(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return __lsx_vssub_bu(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return __lsx_vmax_bu(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return __lsx_vmin_bu(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return 0 == __lsx_vpickve2gr_w(__lsx_vmskltz_b(*this), 0); } - simdjson_inline bool bits_not_set_anywhere() const { return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(*this), 0); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { - return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(__lsx_vand_v(*this, bits)), 0); - } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(__lsx_vsrli_b(*this, N)); } - template - simdjson_inline simd8 shl() const { return simd8(__lsx_vslli_b(*this, N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "LSX kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - template simd8 get() const { return idx < NUM_CHUNKS ? chunks[idx] : simd8(); } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - - simdjson_inline uint64_t compress(uint64_t mask, T * output) const { - uint16_t mask1 = uint16_t(mask); - uint16_t mask2 = uint16_t(mask >> 16); - uint16_t mask3 = uint16_t(mask >> 32); - uint16_t mask4 = uint16_t(mask >> 48); - __m128i zcnt = __lsx_vpcnt_h(__m128i(v2u64{~mask, 0})); - uint64_t zcnt1 = __lsx_vpickve2gr_hu(zcnt, 0); - uint64_t zcnt2 = __lsx_vpickve2gr_hu(zcnt, 1); - uint64_t zcnt3 = __lsx_vpickve2gr_hu(zcnt, 2); - uint64_t zcnt4 = __lsx_vpickve2gr_hu(zcnt, 3); - uint8_t *voutput = reinterpret_cast(output); - // There should be a critical value which processes in scaler is faster. - if (zcnt1) - this->chunks[0].compress(mask1, reinterpret_cast(voutput)); - voutput += zcnt1; - if (zcnt2) - this->chunks[1].compress(mask2, reinterpret_cast(voutput)); - voutput += zcnt2; - if (zcnt3) - this->chunks[2].compress(mask3, reinterpret_cast(voutput)); - voutput += zcnt3; - if (zcnt4) - this->chunks[3].compress(mask4, reinterpret_cast(voutput)); - voutput += zcnt4; - return reinterpret_cast(voutput) - reinterpret_cast(output); - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr+sizeof(simd8)*0); - this->chunks[1].store(ptr+sizeof(simd8)*1); - this->chunks[2].store(ptr+sizeof(simd8)*2); - this->chunks[3].store(ptr+sizeof(simd8)*3); - } - - simdjson_inline uint64_t to_bitmask() const { - __m128i mask1 = __lsx_vmskltz_b(this->chunks[0]); - __m128i mask2 = __lsx_vmskltz_b(this->chunks[1]); - __m128i mask3 = __lsx_vmskltz_b(this->chunks[2]); - __m128i mask4 = __lsx_vmskltz_b(this->chunks[3]); - mask1 = __lsx_vilvl_h(mask2, mask1); - mask2 = __lsx_vilvl_h(mask4, mask3); - return __lsx_vpickve2gr_du(__lsx_vilvl_w(mask2, mask1), 0); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64 &other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_LSX_SIMD_H -/* end file simdjson/lsx/simd.h */ -/* including simdjson/lsx/stringparsing_defs.h: #include "simdjson/lsx/stringparsing_defs.h" */ -/* begin file simdjson/lsx/stringparsing_defs.h */ -#ifndef SIMDJSON_LSX_STRINGPARSING_DEFS_H -#define SIMDJSON_LSX_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + sizeof(v0)); - v0.store(dst); - v1.store(dst + sizeof(v0)); - - // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on LSX; therefore, we - // smash them together into a 64-byte mask and get the bitmask from there. - uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; -} - - -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_LSX_STRINGPARSING_DEFS_H -/* end file simdjson/lsx/stringparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 -/* end file simdjson/lsx/begin.h */ -/* including simdjson/generic/amalgamated.h for lsx: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for lsx */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for lsx: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for lsx */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for lsx */ -/* including simdjson/generic/jsoncharutils.h for lsx: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for lsx */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for lsx */ -/* including simdjson/generic/atomparsing.h for lsx: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for lsx */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace lsx { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for lsx */ -/* including simdjson/generic/dom_parser_implementation.h for lsx: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for lsx */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace lsx -} // namespace simdjson - -namespace simdjson { -namespace lsx { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for lsx */ -/* including simdjson/generic/implementation_simdjson_result_base.h for lsx: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for lsx */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for lsx */ -/* including simdjson/generic/numberparsing.h for lsx: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for lsx */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace lsx { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for lsx */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for lsx: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for lsx */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for lsx */ -/* end file simdjson/generic/amalgamated.h for lsx */ -/* including simdjson/lsx/end.h: #include "simdjson/lsx/end.h" */ -/* begin file simdjson/lsx/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT -/* undefining SIMDJSON_IMPLEMENTATION from "lsx" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/lsx/end.h */ - -#endif // SIMDJSON_LSX_H -/* end file simdjson/lsx.h */ -#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(rvv_vls) -/* including simdjson/rvv-vls.h: #include "simdjson/rvv-vls.h" */ -/* begin file simdjson/rvv-vls.h */ -#ifndef SIMDJSON_RVV_VLS_H -#define SIMDJSON_RVV_VLS_H - - -/* including simdjson/rvv-vls/begin.h: #include "simdjson/rvv-vls/begin.h" */ -/* begin file simdjson/rvv-vls/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "rvv_vls" */ -#define SIMDJSON_IMPLEMENTATION rvv_vls -/* including simdjson/rvv-vls/base.h: #include "simdjson/rvv-vls/base.h" */ -/* begin file simdjson/rvv-vls/base.h */ -#ifndef SIMDJSON_RVV_VLS_BASE_H -#define SIMDJSON_RVV_VLS_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -/** - * RVV-VLS implementation. - */ -namespace rvv_vls { - -class implementation; - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_BASE_H -/* end file simdjson/rvv-vls/base.h */ -/* including simdjson/rvv-vls/intrinsics.h: #include "simdjson/rvv-vls/intrinsics.h" */ -/* begin file simdjson/rvv-vls/intrinsics.h */ -#ifndef SIMDJSON_RVV_VLS_INTRINSICS_H -#define SIMDJSON_RVV_VLS_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#define simdutf_vrgather_u8m1x2(tbl, idx) \ - __riscv_vcreate_v_u8m1_u8m2( \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m2_u8m1(idx, 0), \ - __riscv_vsetvlmax_e8m1()), \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m2_u8m1(idx, 1), \ - __riscv_vsetvlmax_e8m1())) - -#define simdutf_vrgather_u8m1x4(tbl, idx) \ - __riscv_vcreate_v_u8m1_u8m4( \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m4_u8m1(idx, 0), \ - __riscv_vsetvlmax_e8m1()), \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m4_u8m1(idx, 1), \ - __riscv_vsetvlmax_e8m1()), \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m4_u8m1(idx, 2), \ - __riscv_vsetvlmax_e8m1()), \ - __riscv_vrgather_vv_u8m1(tbl, __riscv_vget_v_u8m4_u8m1(idx, 3), \ - __riscv_vsetvlmax_e8m1())) - -#if __riscv_zbc -#include -#endif - -#endif // SIMDJSON_RVV_VLS_INTRINSICS_H -/* end file simdjson/rvv-vls/intrinsics.h */ -/* including simdjson/rvv-vls/bitmanipulation.h: #include "simdjson/rvv-vls/bitmanipulation.h" */ -/* begin file simdjson/rvv-vls/bitmanipulation.h */ -#ifndef SIMDJSON_RVV_VLS_BITMANIPULATION_H -#define SIMDJSON_RVV_VLS_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { - -// We sometimes call trailing_zero on inputs that are zero, -// but the algorithms do not end up using the returned value. -// Sadly, sanitizers are not smart enough to figure it out. -SIMDJSON_NO_SANITIZE_UNDEFINED -// This function can be used safely even if not all bytes have been -// initialized. -// See issue https://github.com/simdjson/simdjson/issues/1965 -SIMDJSON_NO_SANITIZE_MEMORY -simdjson_inline int trailing_zeroes(uint64_t input_num) { - return __builtin_ctzll(input_num); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num-1); -} - -/* result might be undefined when input_num is zero */ -simdjson_inline int leading_zeroes(uint64_t input_num) { - return __builtin_clzll(input_num); -} - -simdjson_inline long long int count_ones(uint64_t input_num) { - return __builtin_popcountll(input_num); -} - - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_BITMANIPULATION_H -/* end file simdjson/rvv-vls/bitmanipulation.h */ -/* including simdjson/rvv-vls/bitmask.h: #include "simdjson/rvv-vls/bitmask.h" */ -/* begin file simdjson/rvv-vls/bitmask.h */ -#ifndef SIMDJSON_RVV_VLS_BITMASK_H -#define SIMDJSON_RVV_VLS_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { - -// -// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. -// -// For example, prefix_xor(00100100) == 00011100 -// -simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { -#if __riscv_zbc - return __riscv_clmul_64(bitmask, ~(uint64_t)0); -#elif __riscv_zvbc - return __riscv_vmv_x(__riscv_vclmul(__riscv_vmv_s_x_u64m1(bitmask, 1), ~(uint64_t)0, 1)); -#else - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; -#endif - return bitmask; -} - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_BITMASK_H - -/* end file simdjson/rvv-vls/bitmask.h */ -/* including simdjson/rvv-vls/simd.h: #include "simdjson/rvv-vls/simd.h" */ -/* begin file simdjson/rvv-vls/simd.h */ -#ifndef SIMDJSON_RVV_VLS_SIMD_H -#define SIMDJSON_RVV_VLS_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace simd { - -#if __riscv_v_fixed_vlen >= 512 - static constexpr size_t VL8 = 512/8; - using vint8_t = vint8m1_t __attribute__((riscv_rvv_vector_bits(512))); - using vuint8_t = vuint8m1_t __attribute__((riscv_rvv_vector_bits(512))); - using vbool_t = vbool8_t __attribute__((riscv_rvv_vector_bits(512/8))); - using vbitmask_t = uint64_t; -#else - static constexpr size_t VL8 = __riscv_v_fixed_vlen/8; - using vint8_t = vint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen))); - using vuint8_t = vuint8m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen))); - using vbool_t = vbool8_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen/8))); - #if __riscv_v_fixed_vlen == 128 - using vbitmask_t = uint16_t; - #elif __riscv_v_fixed_vlen == 256 - using vbitmask_t = uint32_t; - #endif -#endif - -#if __riscv_v_fixed_vlen == 128 - using vuint8x64_t = vuint8m4_t __attribute__((riscv_rvv_vector_bits(512))); - using vboolx64_t = vbool2_t __attribute__((riscv_rvv_vector_bits(512/8))); -#elif __riscv_v_fixed_vlen == 256 - using vuint8x64_t = vuint8m2_t __attribute__((riscv_rvv_vector_bits(512))); - using vboolx64_t = vbool4_t __attribute__((riscv_rvv_vector_bits(512/8))); -#else - using vuint8x64_t = vuint8m1_t __attribute__((riscv_rvv_vector_bits(512))); - using vboolx64_t = vbool8_t __attribute__((riscv_rvv_vector_bits(512/8))); -#endif - - template - struct simd8; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8 { - vbool_t value; - using bitmask_t = vbitmask_t; - static constexpr int SIZE = sizeof(value); - - simdjson_inline simd8(const vbool_t _value) : value(_value) {} - simdjson_inline simd8() : simd8(__riscv_vmclr_m_b8(VL8)) {} - simdjson_inline simd8(bool _value) : simd8(splat(_value)) {} - - simdjson_inline operator const vbool_t&() const { return value; } - simdjson_inline operator vbool_t&() { return value; } - - static simdjson_inline simd8 splat(bool _value) { - return __riscv_vreinterpret_b8(__riscv_vmv_v_x_u64m1(((uint64_t)!_value)-1, 1)); - } - - simdjson_inline vbitmask_t to_bitmask() const { -#if __riscv_v_fixed_vlen == 128 - return __riscv_vmv_x(__riscv_vreinterpret_u16m1(value)); -#elif __riscv_v_fixed_vlen == 256 - return __riscv_vmv_x(__riscv_vreinterpret_u32m1(value)); -#else - return __riscv_vmv_x(__riscv_vreinterpret_u64m1(value)); -#endif - } - - // Bit operations - simdjson_inline simd8 operator|(const simd8 other) const { return __riscv_vmor(*this, other, VL8); } - simdjson_inline simd8 operator&(const simd8 other) const { return __riscv_vmand(*this, other, VL8); } - simdjson_inline simd8 operator^(const simd8 other) const { return __riscv_vmxor(*this, other, VL8); } - simdjson_inline simd8 bit_andnot(const simd8 other) const { return __riscv_vmandn(other, *this, VL8); } - simdjson_inline simd8 operator~() const { return __riscv_vmnot(*this, VL8); } - simdjson_inline simd8& operator|=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline simd8& operator&=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline simd8& operator^=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Unsigned bytes - template<> - struct simd8 { - - vuint8_t value; - static constexpr int SIZE = sizeof(value); - - simdjson_inline simd8(const vuint8_t _value) : value(_value) {} - simdjson_inline simd8() : simd8(zero()) {} - simdjson_inline simd8(const uint8_t values[VL8]) : simd8(load(values)) {} - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - simdjson_inline simd8(simd8 mask) : value(__riscv_vmerge_vxm_u8m1(zero(), -1, (vbool_t)mask, VL8)) {} - - simdjson_inline operator const vuint8_t&() const { return this->value; } - simdjson_inline operator vuint8_t&() { return this->value; } - - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(vuint8_t{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - static simdjson_inline vuint8_t splat(uint8_t _value) { return __riscv_vmv_v_x_u8m1(_value, VL8); } - static simdjson_inline vuint8_t zero() { return splat(0); } - static simdjson_inline vuint8_t load(const uint8_t values[VL8]) { return __riscv_vle8_v_u8m1(values, VL8); } - - // Bit operations - simdjson_inline simd8 operator|(const simd8 other) const { return __riscv_vor_vv_u8m1( value, other, VL8); } - simdjson_inline simd8 operator&(const simd8 other) const { return __riscv_vand_vv_u8m1( value, other, VL8); } - simdjson_inline simd8 operator^(const simd8 other) const { return __riscv_vxor_vv_u8m1( value, other, VL8); } - simdjson_inline simd8 operator~() const { return __riscv_vnot_v_u8m1(value, VL8); } -#if __riscv_zvbb - simdjson_inline simd8 bit_andnot(const simd8 other) const { return __riscv_vandn_vv_u8m1(other, value, VL8); } -#else - simdjson_inline simd8 bit_andnot(const simd8 other) const { return other & ~*this; } -#endif - simdjson_inline simd8& operator|=(const simd8 other) { value = *this | other; return *this; } - simdjson_inline simd8& operator&=(const simd8 other) { value = *this & other; return *this; } - simdjson_inline simd8& operator^=(const simd8 other) { value = *this ^ other; return *this; } - - simdjson_inline simd8 operator==(const simd8 other) const { return __riscv_vmseq(value, other, VL8); } - simdjson_inline simd8 operator==(uint8_t other) const { return __riscv_vmseq(value, other, VL8); } - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return __riscv_vslideup(__riscv_vslidedown(prev_chunk, VL8-N, VL8), value, N, VL8); - } - - // Store to array - simdjson_inline void store(uint8_t dst[VL8]) const { return __riscv_vse8(dst, value, VL8); } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return __riscv_vsaddu(value, other, VL8); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return __riscv_vssubu(value, other, VL8); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return __riscv_vadd(value, other, VL8); } - simdjson_inline simd8 operator-(const simd8 other) const { return __riscv_vsub(value, other, VL8); } - simdjson_inline simd8& operator+=(const simd8 other) { value = *this + other; return *this; } - simdjson_inline simd8& operator-=(const simd8 other) { value = *this - other; return *this; } - - // Order-specific operations - simdjson_inline simd8 operator<=(const simd8 other) const { return __riscv_vmsleu(value, other, VL8); } - simdjson_inline simd8 operator>=(const simd8 other) const { return __riscv_vmsgeu(value, other, VL8); } - simdjson_inline simd8 operator<(const simd8 other) const { return __riscv_vmsltu(value, other, VL8); } - simdjson_inline simd8 operator>(const simd8 other) const { return __riscv_vmsgtu(value, other, VL8); } - - // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. - simdjson_inline simd8 gt_bits(const simd8 other) const { return simd8(*this > other); } - // Same as <, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. - simdjson_inline simd8 lt_bits(const simd8 other) const { return simd8(*this < other); } - - // Bit-specific operations - simdjson_inline bool any_bits_set_anywhere() const { - return __riscv_vfirst(__riscv_vmsne(value, 0, VL8), VL8) >= 0; - } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } - template - simdjson_inline simd8 shr() const { return __riscv_vsrl(value, N, VL8); } - template - simdjson_inline simd8 shl() const { return __riscv_vsll(value, N, VL8); } - - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return __riscv_vrgather(lookup_table, value, VL8); - } - - // compress inactive elements, to match AVX-512 behavior - template - simdjson_inline void compress(vbitmask_t mask, L * output) const { - mask = (vbitmask_t)~mask; -#if __riscv_v_fixed_vlen == 128 - vbool8_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u16m1(mask, 1)); -#elif __riscv_v_fixed_vlen == 256 - vbool8_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u32m1(mask, 1)); -#else - vbool8_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u64m1(mask, 1)); -#endif - __riscv_vse8_v_u8m1(output, __riscv_vcompress(value, m, VL8), count_ones(mask)); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 { - vint8_t value; - static constexpr int SIZE = sizeof(value); - - simdjson_inline simd8(const vint8_t _value) : value(_value) {} - simdjson_inline simd8() : simd8(zero()) {} - simdjson_inline simd8(const int8_t values[VL8]) : simd8(load(values)) {} - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - - simdjson_inline operator const vint8_t&() const { return this->value; } - simdjson_inline operator vint8_t&() { return this->value; } - - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(vint8_t{ - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - }) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10,v11,v12,v13,v14,v15 - ); - } - - static simdjson_inline vint8_t splat(int8_t _value) { return __riscv_vmv_v_x_i8m1(_value, VL8); } - static simdjson_inline vint8_t zero() { return splat(0); } - static simdjson_inline vint8_t load(const int8_t values[VL8]) { return __riscv_vle8_v_i8m1(values, VL8); } - - - simdjson_inline void store(int8_t dst[VL8]) const { return __riscv_vse8(dst, value, VL8); } - - // Explicit conversion to/from unsigned - simdjson_inline explicit simd8(const vuint8_t other): simd8(__riscv_vreinterpret_i8m1(other)) {} - simdjson_inline explicit operator simd8() const { return __riscv_vreinterpret_u8m1(value); } - - // Math - simdjson_inline simd8 operator+(const simd8 other) const { return __riscv_vadd(value, other, VL8); } - simdjson_inline simd8 operator-(const simd8 other) const { return __riscv_vsub(value, other, VL8); } - simdjson_inline simd8& operator+=(const simd8 other) { value = *this + other; return *this; } - simdjson_inline simd8& operator-=(const simd8 other) { value = *this - other; return *this; } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val( const simd8 other) const { return __riscv_vmax( value, other, VL8); } - simdjson_inline simd8 min_val( const simd8 other) const { return __riscv_vmin( value, other, VL8); } - simdjson_inline simd8 operator>( const simd8 other) const { return __riscv_vmsgt(value, other, VL8); } - simdjson_inline simd8 operator<( const simd8 other) const { return __riscv_vmslt(value, other, VL8); } - simdjson_inline simd8 operator==(const simd8 other) const { return __riscv_vmseq(value, other, VL8); } - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return __riscv_vslideup(__riscv_vslidedown(prev_chunk, VL8-N, VL8), value, N, VL8); - } - - // Perform a lookup assuming no value is larger than 16 - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return __riscv_vrgather(lookup_table, value, VL8); - } - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - template - struct simd8x64; - template<> - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - vuint8x64_t value; - -#if __riscv_v_fixed_vlen >= 512 - template simd8 get() const { return value; } -#else - template simd8 get() const { return __riscv_vget_u8m1(value, idx); } -#endif - - simdjson_inline operator const vuint8x64_t&() const { return this->value; } - simdjson_inline operator vuint8x64_t&() { return this->value; } - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - -#if __riscv_v_fixed_vlen == 128 - simdjson_inline simd8x64(const uint8_t *ptr, size_t n = 64) : value(__riscv_vle8_v_u8m4(ptr, n)) {} -#elif __riscv_v_fixed_vlen == 256 - simdjson_inline simd8x64(const uint8_t *ptr, size_t n = 64) : value(__riscv_vle8_v_u8m2(ptr, n)) {} -#else - simdjson_inline simd8x64(const uint8_t *ptr, size_t n = 64) : value(__riscv_vle8_v_u8m1(ptr, n)) {} -#endif - - simdjson_inline void store(uint8_t ptr[64]) const { - __riscv_vse8(ptr, value, 64); - } - - simdjson_inline bool is_ascii() const { -#if __riscv_v_fixed_vlen == 128 - return __riscv_vfirst(__riscv_vmslt(__riscv_vreinterpret_i8m4(value), 0, 64), 64) < 0; -#elif __riscv_v_fixed_vlen == 256 - return __riscv_vfirst(__riscv_vmslt(__riscv_vreinterpret_i8m2(value), 0, 64), 64) < 0; -#else - return __riscv_vfirst(__riscv_vmslt(__riscv_vreinterpret_i8m1(value), 0, 64), 64) < 0; -#endif - } - - // compress inactive elements, to match AVX-512 behavior - simdjson_inline uint64_t compress(uint64_t mask, uint8_t * output) const { - mask = ~mask; -#if __riscv_v_fixed_vlen == 128 - vboolx64_t m = __riscv_vreinterpret_b2(__riscv_vmv_s_x_u64m1(mask, 1)); -#elif __riscv_v_fixed_vlen == 256 - vboolx64_t m = __riscv_vreinterpret_b4(__riscv_vmv_s_x_u64m1(mask, 1)); -#else - vboolx64_t m = __riscv_vreinterpret_b8(__riscv_vmv_s_x_u64m1(mask, 1)); -#endif - size_t cnt = count_ones(mask); - __riscv_vse8(output, __riscv_vcompress(value, m, 64), cnt); - return cnt; - } - - simdjson_inline uint64_t eq(const uint8_t m) const { - return __riscv_vmv_x(__riscv_vreinterpret_u64m1(__riscv_vmseq(value, m, 64))); - } - - simdjson_inline uint64_t lteq(const uint8_t m) const { - return __riscv_vmv_x(__riscv_vreinterpret_u64m1(__riscv_vmsleu(value, m, 64))); - } - }; // struct simd8x64 - -} // namespace simd -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_SIMD_H -/* end file simdjson/rvv-vls/simd.h */ -/* including simdjson/rvv-vls/stringparsing_defs.h: #include "simdjson/rvv-vls/stringparsing_defs.h" */ -/* begin file simdjson/rvv-vls/stringparsing_defs.h */ -#ifndef SIMDJSON_RVV_VLS_STRINGPARSING_DEFS_H -#define SIMDJSON_RVV_VLS_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { - -using namespace simd; - -// Holds backslashes and quotes locations. -struct backslash_and_quote { -public: - static constexpr uint64_t BYTES_PROCESSED = sizeof(simd8); - simdjson_inline backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint64_t bs_bits; - uint64_t quote_bits; -}; // struct backslash_and_quote - -simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - v.store(dst); - return { (v == '\\').to_bitmask(), (v == '"').to_bitmask() }; -} - - -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_RVV_VLS_STRINGPARSING_DEFS_H -/* end file simdjson/rvv-vls/stringparsing_defs.h */ -/* including simdjson/rvv-vls/numberparsing_defs.h: #include "simdjson/rvv-vls/numberparsing_defs.h" */ -/* begin file simdjson/rvv-vls/numberparsing_defs.h */ -#ifndef SIMDJSON_RVV_VLS_NUMBERPARSING_DEFS_H -#define SIMDJSON_RVV_VLS_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#ifdef JSON_TEST_NUMBERS // for unit testing -void found_invalid_number(const uint8_t *buf); -void found_integer(int64_t result, const uint8_t *buf); -void found_unsigned_integer(uint64_t result, const uint8_t *buf); -void found_float(double result, const uint8_t *buf); -#endif - -namespace simdjson { -namespace rvv_vls { -namespace numberparsing { - -// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const char *chars) { - uint64_t val; -#if __riscv_misaligned_fast - memcpy(&val, chars, sizeof(uint64_t)); -#else - val = __riscv_vmv_x(__riscv_vreinterpret_u64m1(__riscv_vlmul_ext_u8m1(__riscv_vle8_v_u8mf2((uint8_t*)chars, 8)))); -#endif - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); -} - -/** @private */ -static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { - return parse_eight_digits_unrolled(reinterpret_cast(chars)); -} - -/** @private */ -simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); - return answer; -} - -} // namespace numberparsing -} // namespace rvv_vls -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_RVV_VLS_NUMBERPARSING_DEFS_H -/* end file simdjson/rvv-vls/numberparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 -/* end file simdjson/rvv-vls/begin.h */ -/* including simdjson/generic/amalgamated.h for rvv_vls: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for rvv_vls */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for rvv_vls: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for rvv_vls */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ -/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ -/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_RVV_VLS */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { - -struct open_container; -class dom_parser_implementation; - -/** - * The type of a JSON number - */ -enum class number_type { - floating_point_number=1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer, /// a positive integer larger or equal to 1<<63 - big_integer /// a big integer that does not fit in a 64-bit word -}; - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for rvv_vls */ -/* including simdjson/generic/jsoncharutils.h for rvv_vls: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for rvv_vls */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace jsoncharutils { - -// return non-zero if not a structural or whitespace char -// zero otherwise -simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; -} - -simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; -} - -// returns a value with the high 16 bits set if not valid -// otherwise returns the conversion of the 4 hex digits at src into the bottom -// 16 bits of the 32-bit return register -// -// see -// https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ -static inline uint32_t hex_to_u32_nocheck( - const uint8_t *src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; -} - -// given a code point cp, writes to c -// the utf-8 code, outputting the length in -// bytes, if the length is zero, the code point -// is invalid -// -// This can possibly be made faster using pdep -// and clz and table lookups, but JSON documents -// have few escaped code points, and the following -// function looks cheap. -// -// Note: we assume that surrogates are treated separately -// -simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r -} - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm -// this is a slow emulation routine for 32-bit -// -static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} -static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif - -} // namespace jsoncharutils -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for rvv_vls */ -/* including simdjson/generic/atomparsing.h for rvv_vls: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for rvv_vls */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { -namespace rvv_vls { -namespace { -/// @private -namespace atomparsing { - -// The string_to_uint32 is exclusively used to map literal strings to 32-bit values. -// We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot -// be certain that the character pointer will be properly aligned. -// You might think that using memcpy makes this function expensive, but you'd be wrong. -// All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); -// to the compile-time constant 1936482662. -simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - -// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory -simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } - - -// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. -// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. -simdjson_warn_unused -simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); -} - -// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { - uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); - std::memcpy(&srcval, src, sizeof(uint64_t)); - - return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); -} - -// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. -// -// 'atom' must consist of only lowercase letters. -simdjson_warn_unused -simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { - return ((src[0] | 0x20) ^ atom[0]) // - | ((src[1] | 0x20) ^ atom[1]) // - | ((src[2] | 0x20) ^ atom[2]); -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src) { - return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src+1, "alse"); } - else { return false; } -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } -} - -#if SIMDJSON_ENABLE_NAN_INF -// "nan" is 3 bytes; we check characters and then verify the next -// character is structural or whitespace. We accept both "nan" and "NaN". -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; -} - -// checks that the next four characters of a string are 'nan"', where the 'nan' -// is checked in a case-insensitive way. -simdjson_warn_unused -simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { - return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { - if (len > 3) { return is_valid_nan_atom(src); } - if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } - return false; -} - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - if(is_short_inf) return true; - - // Check for 'infinity' (any capitalization) - return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; -} - -simdjson_warn_unused -simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { - bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; - if(is_short_inf) return true; - - return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; -} - - -// This function will accept any case-insensitive 3-character spelling of -// infinity: 'inf', 'INF', and 'Inf' are all accepted. -// -// Any capitalization of 'infinity' is also accepted. -simdjson_warn_unused -simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { - if (len > 8) { return is_valid_inf_atom(src); } - if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { - return true; - } - if (len > 3) { - return (str3ncmp_case_insensitive(src, "inf") - | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; - } - if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } - return false; -} -#endif // SIMDJSON_ENABLE_NAN_INF - -} // namespace atomparsing -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for rvv_vls */ -/* including simdjson/generic/dom_parser_implementation.h for rvv_vls: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for rvv_vls */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { - -// expectation: sizeof(open_container) = 64/8. -struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope -}; // struct open_container - -static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits"); - -class dom_parser_implementation final : public internal::dom_parser_implementation { -public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t *buf{}; - /** Length passed to stage 1 */ - size_t len{0}; - /** Document passed to stage 2 */ - dom::document *doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - dom_parser_implementation(const dom_parser_implementation &) = delete; - dom_parser_implementation &operator=(const dom_parser_implementation &) = delete; - - simdjson_warn_unused error_code parse(const uint8_t *buf, size_t len, dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; - stage2_next_function_t stage2_next_function(bool bounded) const noexcept final; - simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; -private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - -}; - -} // namespace rvv_vls -} // namespace simdjson - -namespace simdjson { -namespace rvv_vls { - -inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; - -// Leaving these here so they can be inlined if so desired -inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if(capacity > SIMDJSON_MAXSIZE_BYTES || capacity > SIZE_MAX - 63) { return CAPACITY; } - // Stage 1 index output - size_t rounded_capacity = SIMDJSON_ROUNDUP_N(capacity, 64); - if(rounded_capacity + 9 < rounded_capacity) { - return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity - } - size_t max_structures = rounded_capacity + 9; - if(max_structures > SIZE_MAX / sizeof(uint32_t)) { return CAPACITY; } - structural_indexes.reset( new (std::nothrow) uint32_t[max_structures] ); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; -} - -inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; -} - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for rvv_vls */ -/* including simdjson/generic/implementation_simdjson_result_base.h for rvv_vls: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for rvv_vls */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { - -// This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair -// so we can avoid inlining errors -// TODO reconcile these! -/** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ -template -struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T &value, error_code &error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; - - /** - * The error. - */ - simdjson_warn_unused simdjson_inline error_code error() const noexcept; - - /** - * Whether there is a value. - */ - simdjson_warn_unused simdjson_inline bool has_value() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& operator*() & noexcept(false); - simdjson_inline T&& operator*() && noexcept(false); - /** - * Arrow operator to access members of the contained value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T* operator->() noexcept(false); - simdjson_inline const T* operator->() const noexcept(false); - - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T&&() && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - - using value_type = T; - using error_type = error_code; - -protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{UNINITIALIZED}; /** Users should never directly access 'second'. **/ -}; // struct implementation_simdjson_result_base - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for rvv_vls */ -/* including simdjson/generic/numberparsing.h for rvv_vls: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for rvv_vls */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { -namespace rvv_vls { -namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) -#endif - -namespace { - -// Convert a mantissa, an exponent and a sign bit into an ieee64 double. -// The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). -// The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; -} - -// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). -// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). -// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. -simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { - float f; - mantissa &= ~(uint32_t(1) << 23); - mantissa |= real_exponent << 23; - mantissa |= ((static_cast(negative)) << 31); - std::memcpy(&f, &mantissa, sizeof(f)); - return f; -} - -// Attempts to compute i * 10^(power) exactly; and if "negative" is -// true, negate the result. -// This function will only work in some cases, when it does not work, success is -// set to false. This should work *most of the time* (like 99% of the time). -// We assume that power is in the [smallest_power, -// largest_power] interval: the caller is responsible for this check. -simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if(i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximately equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; -} - -// Attempts to compute i * 10^(power) as a binary32 (float) value; and if -// "negative" is true, negate the result. -// -// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at -// binary32. It is adapted from fast_float -// (https://github.com/fastfloat/fast_float), where the routine is written once -// and instantiated for each binary format. We only need 24 bits of mantissa -// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits -// (plus one bit that may be lost to the "upperbit" shift), and we shift the -// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). -// -// The power-of-five table is shared with the binary64 code: the useful range -// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict -// subset of [smallest_power, largest_power]. -// -// The function returns false when the result would be infinite: simdjson -// refuses to parse infinite values, so the caller reports an error. Unlike -// compute_float_64, a false return never means "try harder": the accuracy of -// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, -// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). -// The caller must still fall back to a slow path when the 64-bit mantissa i -// was truncated (more than 19 significant digits). -// -// We assume that power is in the [smallest_power, largest_power] interval: the -// caller is responsible for this check. -simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { - // Powers of ten that are exactly representable as binary32 values: 10^k is - // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). - static constexpr float power_of_ten_float[] = { - 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, - 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; - // The range of powers of ten that a non-zero, finite binary32 value can be - // built from. Anything smaller rounds to zero, anything larger is infinite. - // These are the smallest_power_of_ten()/largest_power_of_ten() constants that - // fast_float uses for binary32. - constexpr int smallest_power_binary32 = -65; - constexpr int largest_power_binary32 = 38; - - // We start with the fast path described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 10 && i <= 16777215) -#else - if (-10 <= power && power <= 10 && i <= 16777215) -#endif - { - // Convert the integer into a float. This is lossless since - // 0 <= i <= 2^24 - 1. - d = float(i); - // Both d and the power of ten are exactly representable as binary32 - // values, so the product (or quotient) is correctly rounded. - if (power < 0) { - d = d / power_of_ten_float[-power]; - } else { - d = d * power_of_ten_float[power]; - } - if (negative) { - d = -d; - } - return true; - } - - // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. - // It needs i > 0 (so that the leading bit of i can be normalized), so we - // handle i == 0 separately. We also handle the powers of ten that are so - // small (or so large) that the answer is zero (or infinite) whatever the - // mantissa is. - if (i == 0 || power < smallest_power_binary32) { - d = negative ? -0.0f : 0.0f; - return true; - } - if (power > largest_power_binary32) { - // We have, for sure, an infinite value. - return false; - } - - // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). - // The 128 comes from the ieee32 standard (the minimal exponent is -127). - // The 63 comes from the fact that we use a 64-bit word. - // See compute_float_64 for a discussion of the magical 152170 + 65536. - int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - // We want the most significant 64 bits of the product i * 5**power. It is - // safe to index the table because - // smallest_power <= smallest_power_binary32 <= power - // <= largest_power_binary32 <= largest_power. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); -#else - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); -#endif - - // Unless the least significant 38 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 26 bits are - // exact and no further work is needed. Having 26 bits is necessary because - // we need 24 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) - if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { - // The truncated multiplication was not accurate enough; use the next 64 - // bits of the power of five to refine it. See compute_float_64 for a - // detailed discussion. -#if SIMDJSON_STATIC_REFLECTION - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); -#else - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); -#endif - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { firstproduct.high++; } - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 24 bits with a leading 1. - // We shift it so that it occupies 25 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<25). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here we have that real_exponent <= 0 so -real_exponent >= 0 - if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0f : 0.0f; - return true; - } - // next line is safe because -real_exponent + 1 < 64 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // As in compute_float_64, rounding up may take us out of the subnormal - // range, so we can only decide after rounding. - real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; - } - // We have to round to even. The "to even" part is only a problem when we are - // right in between two floats, which we guard against. The bounds on the - // power of ten are those of fast_float's - // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: - // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; - // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. - if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { - if((mantissa << (upperbit + 38)) == upper) { - mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<24), unless there was an overflow - if (mantissa >= (uint64_t(1) << 24)) { - mantissa = (uint64_t(1) << 23); - real_exponent++; - } - mantissa &= ~(uint64_t(1) << 23); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 254)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); - return true; -} - -#if SIMDJSON_ENABLE_NAN_INF -// Parses a nan or infinity. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - double inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} - -// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. -simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { - if (atomparsing::is_valid_inf_atom(src)) { - float inf = std::numeric_limits::infinity(); - d = negative ? -inf : inf; - return true; - } - - if (atomparsing::is_valid_nan_atom(src)) { - d = std::numeric_limits::quiet_NaN(); - return true; - } - - return false; -} -#endif - -// We call a fallback floating-point parser that might be slow. Note -// it will accept JSON numbers, but the JSON spec. is more restrictive so -// before you call parse_float_fallback, you need to have validated the input -// string with the JSON grammar. -// It will return an error (false) if the parsed number is infinite. -// The string parsing itself always succeeds. We know that there is at least -// one digit. -static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); -} - -// Same as parse_float_fallback, but for binary32 (float) values. Going through -// the binary64 fallback and then rounding to binary32 would be subject to -// double rounding, so we run the fallback algorithm directly on binary32. -static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { - *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); - // We do not accept infinite values. See the binary64 version above for why we - // do not use std::isfinite. - return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); -} - -// check quickly whether the next 8 chars are made of digits -// at a glance, it looks better than Mula's -// http://0x80.pl/articles/swar-digits-validate.html -simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); -} - -// The same idea for four characters. A block of eight only bites when eight -// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a -// time; taking four of them at once is what makes the tail of a long fraction -// cheap. Adapted from fast_float, credit @aqrit. -simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { - uint32_t val; - // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. - static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); - std::memcpy(&val, chars, 4); - return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); -} - -// Only call this when is_made_of_four_digits_fast() says the four characters are -// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. -simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { - uint32_t val; - std::memcpy(&val, chars, 4); - val -= 0x30303030; - val = (val * 10) + (val >> 8); - return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; -} - -template -SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_inline bool parse_digit(const uint8_t c, I &i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; -} - -simdjson_inline bool is_digit(const uint8_t c) { - return static_cast(c - '0') <= 9; -} - -// Consumes a run of digits into i, eight at a time while eight are available, -// then four, then one at a time. Overflow is deliberate: the caller counts the -// digits and falls back when there are too many for a 64-bit mantissa. -// -// Long runs of digits are what a float-heavy document is made of: the fraction -// of a binary64 printed to full precision is 15 to 17 digits, and reading those -// one at a time is the single largest cost in parse_double(). Integer parts are -// left alone: they are usually a handful of digits, and there the check for a -// block of eight is wasted work. -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - while (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - // A 4 to 7 digit remainder is the common case once the blocks of eight are - // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. - if (is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, - bool &leading_zero) { - if (!parse_digit(*p, i)) { leading_zero = true; return; } - p++; - leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } -} - -SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // Identifiers, timestamps and counters often have eight digits or more. - // Prior related work: jsonifier parses integers as eight-digit SWAR words - // (str_to_i.hpp, https://github.com/nihilai-collective/Jsonifier). This - // takes one such word with parse_eight_digits_unrolled, the routine - // simdjson uses for long fractions. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } - const uint8_t *const swar_end = p + 8; - while (p < swar_end && is_made_of_four_digits_fast(p)) { - i = i * 10000 + parse_four_digits_unrolled(p); - p += 4; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - while (parse_digit(*p, i)) { p++; } -} - -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; - parse_fraction_digits(p, i); - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; -} - -simdjson_warn_unused simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp+18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp+18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; -} - -simdjson_inline bool check_if_integer(const uint8_t *const src, size_t max_length) { - const uint8_t *const srcend = src + max_length; - bool negative = (*src == '-'); // we can always read at least one character after the '-' - const uint8_t *p = src + uint8_t(negative); - if(p == srcend) { return false; } - if(*p == '0') { - ++p; - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; - } - while(p != srcend && is_digit(*p)) { ++p; } - if(p == srcend) { return true; } - if(jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } - return true; -} - -simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); -} - -} // unnamed namespace - -/** @private */ -inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { - if (parse_float_fallback(src, answer)) { - return SUCCESS; - } - return INVALID_NUMBER(src); -} - -/** @private */ -template -simdjson_warn_unused simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer - // reference to it, it would force it to be stored in memory, preventing the compiler from - // picking it apart and putting into registers. i.e. if we pass it as reference, - // it gets slow. - double d; - error_code error = slow_float_parsing(src, &d); - writer.append_double(d); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; -} - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer); - -// for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds -} - -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. -template -simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { - -#if SIMDJSON_ENABLE_NAN_INF - // By this point, we know that our input does not begin with a digit. We will attempt - // to handle NaN/Infinity. - - double d; - if (compute_nan_inf(p, negative, d)) { - writer.append_double(d); - return SUCCESS; - } -#endif - - return INVALID_NUMBER(src); - } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_decimal_after_separator(src, p, i, exponent) ); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY( parse_exponent(src, p, exponent) ); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) ); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX)+1) { return BIGINT_NUMBER(src); } - WRITE_INTEGER(~i+1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } - } - - // Write unsigned if it does not fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } else { -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(i == 0 && negative) { - // We have to write -0.0 instead of 0 - WRITE_DOUBLE(-0.0, src, writer); - } else { - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); - } -#else - WRITE_INTEGER(negative ? (~i+1) : i, src, writer); -#endif - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; -} - -// Inlineable functions -namespace { - -// This table can be used to characterize the final character of an integer -// string. For JSON structural character and allowable white space characters, -// we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise -// we return NUMBER_ERROR. -// Optimization note: we could easily reduce the size of the table by half (to 128) -// at the cost of an extra branch. -// Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): -static_assert(error_code(uint8_t(NUMBER_ERROR))== NUMBER_ERROR, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(SUCCESS))== SUCCESS, "bad NUMBER_ERROR cast"); -static_assert(error_code(uint8_t(INCORRECT_TYPE))== INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - -const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR}; - -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { - const uint8_t *p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} -// Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { - const uint8_t *p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t *p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; - uint64_t i = 0; - parse_integer_digits(p, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = src; - uint64_t i = 0; - parse_integer_digits(src, i); - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if(*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i+1) : i; -} - -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no loading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - double d; - if (compute_nan_inf(p, negative, d)) { return d; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number into a binary32 (float) value. -// -// This mirrors parse_double, but it rounds to binary32 directly instead of -// rounding to binary64 and then to binary32: the latter is subject to double -// rounding and would not always produce the nearest float. -simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { - -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, the number may be nan or infinity. - // Attempt to compute those, and return on success. - float f; - if (compute_nan_inf(p, negative, f)) { return f; } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - double inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -// Parse a JSON number held inside a JSON string into a binary32 (float) value. -// See parse_float for why we do not simply round parse_double_in_string. -simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - bool leading_zero; - parse_float_integer_digits(p, i, leading_zero); - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { -#if SIMDJSON_ENABLE_NAN_INF - // If there are no leading digits, attempt to parse numbers that are either - // NaN or Infinity - if (atomparsing::is_valid_inf_in_string(src)) { - float inf = std::numeric_limits::infinity(); - return negative ? -inf : inf; - } - - if (atomparsing::is_valid_nan_in_string(src)) { - return std::numeric_limits::quiet_NaN(); - } -#endif - - return INCORRECT_TYPE; - } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t *start_decimal_digits = p; - parse_fraction_digits(p, i); - if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p-start_digits > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits - if (p-start_exp_digits == 0) { return NUMBER_ERROR; } - // 20+ exp digits (e.g., 0e9999999999999999999999) are valid, but exp may - // have wrapped around: leave such numbers to the fallback parser. - overflow = overflow || p-start_exp_digits > 19; - - exponent += exp_neg ? 0-exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - float d; - if (simdjson_likely(!overflow)) { - if (compute_float_32(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; -} - -} // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - -} // namespace numberparsing - -inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - case number_type::big_integer: out << "big integer"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; -} - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for rvv_vls */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for rvv_vls: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for rvv_vls */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { - -// -// internal::implementation_simdjson_result_base inline implementation -// - -template -simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; -} - -template -simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; -} - - -template -simdjson_warn_unused simdjson_inline bool implementation_simdjson_result_base::has_value() const noexcept { - return this->error() == SUCCESS; -} - -#if SIMDJSON_EXCEPTIONS - -template -simdjson_inline T& implementation_simdjson_result_base::operator*() & noexcept(false) { - return this->value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::operator*() && noexcept(false) { - return std::forward>(*this).value(); -} - -template -simdjson_inline T* implementation_simdjson_result_base::operator->() noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - - -template -simdjson_inline const T* implementation_simdjson_result_base::operator->() const noexcept(false) { - if (this->error()) { throw simdjson_error(this->error()); } - return &this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -template -simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { - return std::forward>(*this).take_value(); -} - -#endif // SIMDJSON_EXCEPTIONS - -template -simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; -} - -template -simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; -} - -template -simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); -} - -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept - : first{std::forward(value)}, second{error} {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} -template -simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for rvv_vls */ -/* end file simdjson/generic/amalgamated.h for rvv_vls */ -/* including simdjson/rvv-vls/end.h: #include "simdjson/rvv-vls/end.h" */ -/* begin file simdjson/rvv-vls/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/rvv-vls/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* undefining SIMDJSON_IMPLEMENTATION from "rvv_vls" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/rvv-vls/end.h */ - -#endif // SIMDJSON_RVV_VLS_H -/* end file simdjson/rvv-vls.h */ -#else -#error Unknown SIMDJSON_BUILTIN_IMPLEMENTATION -#endif - -/* undefining SIMDJSON_CONDITIONAL_INCLUDE */ -#undef SIMDJSON_CONDITIONAL_INCLUDE - -#endif // SIMDJSON_BUILTIN_H -/* end file simdjson/builtin.h */ -/* skipped duplicate #include "simdjson/builtin/base.h" */ - -/* including simdjson/generic/builder/dependencies.h: #include "simdjson/generic/builder/dependencies.h" */ -/* begin file simdjson/generic/builder/dependencies.h */ -#ifdef SIMDJSON_CONDITIONAL_INCLUDE -#error simdjson/generic/builder/dependencies.h must be included before defining SIMDJSON_CONDITIONAL_INCLUDE! -#endif - -#ifndef SIMDJSON_GENERIC_BUILDER_DEPENDENCIES_H -#define SIMDJSON_GENERIC_BUILDER_DEPENDENCIES_H - -// Internal headers needed for builder generics. -// All includes not under simdjson/generic/builder must be here! -// Otherwise, amalgamation will fail. -/* skipped duplicate #include "simdjson/concepts.h" */ -/* skipped duplicate #include "simdjson/dom/fractured_json.h" */ -/* including simdjson/annotations.h: #include "simdjson/annotations.h" */ -/* begin file simdjson/annotations.h */ -#ifndef SIMDJSON_ANNOTATIONS_H -#define SIMDJSON_ANNOTATIONS_H - -/** - * @file annotations.h - * @brief Provides compile-time annotations for simdjson structures. - * This header defines annotations that can be applied to data members of structures - * (and to the structures and enumerations themselves) to control how they are - * serialized/deserialized with simdjson. The set of annotations is modelled after - * the attributes of the Rust serde library. - * - * Member annotations: - * - * [[= simdjson::rename<"name">]] use "name" as the JSON key - * [[= simdjson::alias<"a", "b">]] also accept "a" and "b" when deserializing - * [[= simdjson::skip]] never serialize nor deserialize - * [[= simdjson::skip_serializing]] never serialize - * [[= simdjson::skip_deserializing]] never deserialize (keeps its current value) - * [[= simdjson::skip_serializing_if]] do not serialize when pred(value) is true - * [[= simdjson::default_value]] a missing key is not an error - * [[= simdjson::default_from]] a missing key sets the member to factory() - * [[= simdjson::with]] custom (de)serialization via Adapter - * [[= simdjson::flatten]] inline the members of a nested structure - * - * Structure (container) annotations: - * - * [[= simdjson::rename_all]] rename every member - * [[= simdjson::default_value]] no missing key is an error - * [[= simdjson::deny_unknown_fields]] unknown keys are a deserialization error - * [[= simdjson::transparent]] (de)serialize as the single member - * - * Enumeration annotations: rename_all on the enumeration, rename and alias on the - * enumerators (e.g., `enum class color { red [[= simdjson::rename<"RED">]] };`). - * - * This is currently experimental and subject to change (syntax and semantics may evolve). - */ - -#if SIMDJSON_STATIC_REFLECTION - -#include -#include -#include -#include - -namespace simdjson { - -// Structural compile-time string -- char array avoids the pointer-based -// 'reflect_constant failed' that occurs with const char* / string_view members. -template -struct fixed_string { - char data[N]; - - consteval fixed_string(const char (&s)[N]) noexcept { - for (size_t i = 0; i < N; ++i) { data[i] = s[i]; } - } - - consteval std::string_view view() const noexcept { return {data, N - 1}; } - - consteval bool operator==(const fixed_string&) const noexcept = default; -}; - -/** - * Naming conventions for simdjson::rename_all, mirroring serde's rename_all. - * Except for lowercase and uppercase, the C++ identifier is first split into - * words at underscores and at case changes ("userId", "user_id" and "UserId" all - * give the words "user" and "id"; "HTTPServer" gives "HTTP" and "Server"), and - * the words are then joined according to the convention. - */ -enum class case_style { - lowercase, ///< every letter lowercased, nothing else changes: userId -> userid - uppercase, ///< every letter uppercased, nothing else changes: user_id -> USER_ID - pascal_case, ///< UserId - camel_case, ///< userId - snake_case, ///< user_id - screaming_snake_case, ///< USER_ID - kebab_case, ///< user-id - screaming_kebab_case ///< USER-ID -}; - -namespace detail { - template - struct rename_t { - static constexpr auto name = Name; - // Exposed as a pointer and a size: std::meta::extract requires structural types. - static constexpr const char *key_data = Name.data; - static constexpr size_t key_size = Name.view().size(); - }; - template - struct alias_t { - static_assert(sizeof...(Names) > 0, "simdjson::alias requires at least one name"); - static constexpr std::string_view keys[] = {Names.view()...}; - static constexpr const std::string_view *keys_data = keys; - static constexpr size_t keys_count = sizeof...(Names); - }; - struct skip_tag {}; - struct skip_serializing_tag {}; - struct skip_deserializing_tag {}; - template - struct skip_serializing_if_t { - static constexpr auto predicate = Predicate; - }; - struct default_value_tag {}; - template - struct default_from_t { - static constexpr auto factory = Factory; - }; - template - struct with_t { - using adapter = Adapter; - }; - template - struct rename_all_t { - static constexpr case_style style = Style; - }; - struct deny_unknown_fields_tag {}; - struct transparent_tag {}; - struct flatten_tag {}; - - // Predicates usable with skip_serializing_if. - struct is_none_t { - template - constexpr bool operator()(const T& v) const noexcept { return !v; } - }; - struct is_empty_t { - template - constexpr bool operator()(const T& v) const noexcept { return v.empty(); } - }; -} // namespace detail - -// Usage: [[= simdjson::rename<"first_name">]] std::string firstName; -template -inline constexpr detail::rename_t rename{}; - -// Usage: [[= simdjson::alias<"userName", "login">]] std::string user_name; -// The aliases are accepted (in addition to the regular key) when deserializing. -// Serialization always uses the regular key. If the JSON object contains more -// than one of the names, which one is used is unspecified. -template -inline constexpr detail::alias_t alias{}; - -// Usage: [[= simdjson::skip]] int internalCache; -inline constexpr detail::skip_tag skip{}; - -// Usage: [[= simdjson::skip_serializing]] std::string password; -inline constexpr detail::skip_serializing_tag skip_serializing{}; - -// Usage: [[= simdjson::skip_deserializing]] int computed; -// The member is never assigned during deserialization (it keeps its current -// value) and a matching key in the JSON input is treated as unknown. -inline constexpr detail::skip_deserializing_tag skip_deserializing{}; - -// Usage: [[= simdjson::skip_serializing_if]] std::optional x; -// The predicate is called with the member value; when it returns true, the key -// is omitted from the output. -template -inline constexpr detail::skip_serializing_if_t skip_serializing_if{}; - -// Predicate: true for an empty std::optional, a null smart pointer, etc. -inline constexpr detail::is_none_t is_none{}; -// Predicate: true for an empty string or container. -inline constexpr detail::is_empty_t is_empty{}; - -// Usage: [[= simdjson::default_value]] int port = 8080; -// When the key is missing from the JSON input, the member is left untouched -// (with get(), it keeps its default member initializer) instead of reporting -// NO_SUCH_FIELD. Applied to a structure, it applies to all of its members. -inline constexpr detail::default_value_tag default_value{}; - -// Usage: [[= simdjson::default_from]] int port; -// When the key is missing from the JSON input, the member is assigned the -// result of calling the factory (a constexpr callable taking no argument, such -// as a captureless lambda or a pointer to a function). -template -inline constexpr detail::default_from_t default_from{}; - -// Usage: [[= simdjson::with]] std::chrono::system_clock::time_point t; -// Adapter is a type that provides one or both of -// static void serialize(simdjson::builder::string_builder &b, const T &value); -// static simdjson::error_code deserialize(simdjson::ondemand::value &v, T &out); -// (the parameters may also be declared auto&). When one of them is missing, the -// default behaviour is used in that direction. -template -inline constexpr detail::with_t with{}; - -// Usage: [[= simdjson::flatten]] pagination page; -// The members of the nested structure are (de)serialized as if they were members -// of the enclosing structure: {"id":1,"limit":10,"offset":0} rather than -// {"id":1,"page":{"limit":10,"offset":0}}. The nested structure's own annotations -// (rename_all, default_value, ...) apply to its members. -inline constexpr detail::flatten_tag flatten{}; - -// Usage: struct [[= simdjson::rename_all]] S {...}; -// Also applies to enumerations. An explicit rename on a member takes precedence. -template -inline constexpr detail::rename_all_t