Skip to content

Commit 40c6213

Browse files
jkeiserjkeiser
committed
Add parser.load() and load_many() to load files
1 parent d140bc2 commit 40c6213

37 files changed

Lines changed: 492 additions & 263 deletions

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,10 @@ SRCHEADERS_GENERIC=src/generic/numberparsing.h src/generic/stage1_find_marks.h s
6262
SRCHEADERS_ARM64= src/arm64/bitmanipulation.h src/arm64/bitmask.h src/arm64/intrinsics.h src/arm64/numberparsing.h src/arm64/simd.h src/arm64/stage1_find_marks.h src/arm64/stage2_build_tape.h src/arm64/stringparsing.h
6363
SRCHEADERS_HASWELL= src/haswell/bitmanipulation.h src/haswell/bitmask.h src/haswell/intrinsics.h src/haswell/numberparsing.h src/haswell/simd.h src/haswell/stage1_find_marks.h src/haswell/stage2_build_tape.h src/haswell/stringparsing.h
6464
SRCHEADERS_WESTMERE=src/westmere/bitmanipulation.h src/westmere/bitmask.h src/westmere/intrinsics.h src/westmere/numberparsing.h src/westmere/simd.h src/westmere/stage1_find_marks.h src/westmere/stage2_build_tape.h src/westmere/stringparsing.h
65-
SRCHEADERS_SRC=src/isadetection.h src/jsoncharutils.h src/simdprune_tables.h src/jsonioutil.cpp src/implementation.cpp src/stage1_find_marks.cpp src/stage2_build_tape.cpp src/document_parser_callbacks.h
65+
SRCHEADERS_SRC=src/isadetection.h src/jsoncharutils.h src/simdprune_tables.h src/implementation.cpp src/stage1_find_marks.cpp src/stage2_build_tape.cpp src/document_parser_callbacks.h
6666
SRCHEADERS=$(SRCHEADERS_SRC) $(SRCHEADERS_GENERIC) $(SRCHEADERS_ARM64) $(SRCHEADERS_HASWELL) $(SRCHEADERS_WESTMERE)
6767

68-
INCLUDEHEADERS=include/simdjson.h include/simdjson/common_defs.h include/simdjson/internal/jsonformatutils.h include/simdjson/jsonioutil.h include/simdjson/jsonminifier.h include/simdjson/jsonparser.h include/simdjson/padded_string.h include/simdjson/document.h include/simdjson/inline/document.h include/simdjson/document_iterator.h include/simdjson/inline/document_iterator.h include/simdjson/document_stream.h include/simdjson/inline/document_stream.h include/simdjson/implementation.h include/simdjson/parsedjson.h include/simdjson/jsonstream.h include/simdjson/inline/jsonstream.h include/simdjson/portability.h include/simdjson/error.h include/simdjson/inline/error.h include/simdjson/simdjson.h include/simdjson/simdjson_version.h
68+
INCLUDEHEADERS=include/simdjson.h include/simdjson/common_defs.h include/simdjson/internal/jsonformatutils.h include/simdjson/jsonioutil.h include/simdjson/jsonminifier.h include/simdjson/jsonparser.h include/simdjson/padded_string.h include/simdjson/inline/padded_string.h include/simdjson/document.h include/simdjson/inline/document.h include/simdjson/document_iterator.h include/simdjson/inline/document_iterator.h include/simdjson/document_stream.h include/simdjson/inline/document_stream.h include/simdjson/implementation.h include/simdjson/parsedjson.h include/simdjson/jsonstream.h include/simdjson/inline/jsonstream.h include/simdjson/portability.h include/simdjson/error.h include/simdjson/inline/error.h include/simdjson/simdjson.h include/simdjson/simdjson_version.h
6969

7070
ifeq ($(SIMDJSON_TEST_AMALGAMATED_HEADERS),1)
7171
HEADERS=singleheader/simdjson.h

‎README.md‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -161,10 +161,11 @@ document doc = document::parse(padded_string(string("[ 1, 2, 3 ]")));
161161
doc.print_json(cout);
162162
```
163163

164-
You can also load from a file with `get_corpus`:
164+
You can also load from a file with `parser.load()`:
165165

166166
```c++
167-
document doc = document::parse(get_corpus(filename));
167+
document::parser parser;
168+
document doc = parser.load(filename);
168169
doc.print_json(cout);
169170
```
170171

@@ -222,9 +223,8 @@ Here is a simple example, using single header simdjson:
222223
#include "simdjson.cpp"
223224

224225
int parse_file(const char *filename) {
225-
simdjson::padded_string p = simdjson::get_corpus(filename);
226226
simdjson::document::parser parser;
227-
for (const document &doc : parser.parse_many(p)) {
227+
for (const document &doc : parser.load_many(filename)) {
228228
// do something with the document ...
229229
}
230230
}
@@ -242,12 +242,10 @@ copy the files in your project in your include path. You can then include them q
242242
#include "simdjson.cpp"
243243
using namespace simdjson;
244244
int main(int argc, char *argv[]) {
245-
const char * filename = argv[1];
246-
padded_string p = get_corpus(filename);
247-
document::parser parser = build_parsed_json(p); // do the parsing
248-
if( ! parser.is_valid() ) {
249-
std::cout << "not valid" << std::endl;
250-
std::cout << parser.get_error_message() << std::endl;
245+
document::parser parser;
246+
auto [doc, error] = parser.load(argv[1]);
247+
if(error) {
248+
std::cout << "not valid: " << error << std::endl;
251249
} else {
252250
std::cout << "valid" << std::endl;
253251
}

‎amalgamation.sh‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -127,8 +127,8 @@ int main(int argc, char *argv[]) {
127127
std::cerr << "Please specify at least one file name. " << std::endl;
128128
}
129129
const char * filename = argv[1];
130-
simdjson::padded_string p = simdjson::get_corpus(filename);
131-
auto [doc, error] = simdjson::document::parse(p); // do the parsing
130+
simdjson::document::parser parser;
131+
auto [doc, error] = parser.load(filename); // do the parsing
132132
if (error) {
133133
std::cout << "parse failed" << std::endl;
134134
std::cout << "error code: " << error << std::endl;
@@ -142,9 +142,7 @@ int main(int argc, char *argv[]) {
142142
143143
// parse_many
144144
const char * filename2 = argv[2];
145-
simdjson::padded_string p2 = simdjson::get_corpus(filename2);
146-
simdjson::document::parser parser;
147-
for (auto result : parser.parse_many(p2)) {
145+
for (auto result : parser.load_many(filename2)) {
148146
error = result.error;
149147
}
150148
if (error) {

‎benchmark/bench_dom_api.cpp‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ const padded_string EMPTY_ARRAY("[]", 2);
1313

1414
static void twitter_count(State& state) {
1515
// Prints the number of results in twitter.json
16-
document doc = document::parse(get_corpus(JSON_TEST_PATH));
16+
document doc = document::load(JSON_TEST_PATH);
1717
for (auto _ : state) {
1818
uint64_t result_count = doc["search_metadata"]["count"];
1919
if (result_count != 100) { return; }
@@ -23,7 +23,7 @@ BENCHMARK(twitter_count);
2323

2424
static void error_code_twitter_count(State& state) noexcept {
2525
// Prints the number of results in twitter.json
26-
document doc = document::parse(get_corpus(JSON_TEST_PATH));
26+
document doc = document::load(JSON_TEST_PATH);
2727
for (auto _ : state) {
2828
auto [value, error] = doc["search_metadata"]["count"];
2929
if (error) { return; }
@@ -34,7 +34,7 @@ BENCHMARK(error_code_twitter_count);
3434

3535
static void iterator_twitter_count(State& state) {
3636
// Prints the number of results in twitter.json
37-
document doc = document::parse(get_corpus(JSON_TEST_PATH));
37+
document doc = document::load(JSON_TEST_PATH);
3838
for (auto _ : state) {
3939
document::iterator iter(doc);
4040
// uint64_t result_count = doc["search_metadata"]["count"];
@@ -50,7 +50,7 @@ BENCHMARK(iterator_twitter_count);
5050

5151
static void twitter_default_profile(State& state) {
5252
// Count unique users with a default profile.
53-
document doc = document::parse(get_corpus(JSON_TEST_PATH));
53+
document doc = document::load(JSON_TEST_PATH);
5454
for (auto _ : state) {
5555
set<string_view> default_users;
5656
for (document::object tweet : doc["statuses"].as_array()) {
@@ -66,7 +66,7 @@ BENCHMARK(twitter_default_profile);
6666

6767
static void error_code_twitter_default_profile(State& state) noexcept {
6868
// Count unique users with a default profile.
69-
document doc = document::parse(get_corpus(JSON_TEST_PATH));
69+
document doc = document::load(JSON_TEST_PATH);
7070
for (auto _ : state) {
7171
set<string_view> default_users;
7272

@@ -91,7 +91,7 @@ BENCHMARK(error_code_twitter_default_profile);
9191

9292
static void iterator_twitter_default_profile(State& state) {
9393
// Count unique users with a default profile.
94-
document doc = document::parse(get_corpus(JSON_TEST_PATH));
94+
document doc = document::load(JSON_TEST_PATH);
9595
for (auto _ : state) {
9696
set<string_view> default_users;
9797
document::iterator iter(doc);
@@ -128,7 +128,7 @@ BENCHMARK(iterator_twitter_default_profile);
128128

129129
static void twitter_image_sizes(State& state) {
130130
// Count unique image sizes
131-
document doc = document::parse(get_corpus(JSON_TEST_PATH));
131+
document doc = document::load(JSON_TEST_PATH);
132132
for (auto _ : state) {
133133
set<tuple<uint64_t, uint64_t>> image_sizes;
134134
for (document::object tweet : doc["statuses"].as_array()) {
@@ -148,7 +148,7 @@ BENCHMARK(twitter_image_sizes);
148148

149149
static void error_code_twitter_image_sizes(State& state) noexcept {
150150
// Count unique image sizes
151-
document doc = document::parse(get_corpus(JSON_TEST_PATH));
151+
document doc = document::load(JSON_TEST_PATH);
152152
for (auto _ : state) {
153153
set<tuple<uint64_t, uint64_t>> image_sizes;
154154
auto [statuses, error] = doc["statuses"].as_array();
@@ -175,7 +175,7 @@ BENCHMARK(error_code_twitter_image_sizes);
175175

176176
static void iterator_twitter_image_sizes(State& state) {
177177
// Count unique image sizes
178-
document doc = document::parse(get_corpus(JSON_TEST_PATH));
178+
document doc = document::load(JSON_TEST_PATH);
179179
for (auto _ : state) {
180180
set<tuple<uint64_t, uint64_t>> image_sizes;
181181
document::iterator iter(doc);

‎benchmark/benchmarker.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ struct json_stats {
185185
padded_string load_json(const char *filename) {
186186
try {
187187
verbose() << "[verbose] loading " << filename << endl;
188-
padded_string json = simdjson::get_corpus(filename);
188+
padded_string json = padded_string::load(filename);
189189
verbose() << "[verbose] loaded " << filename << " (" << json.size() << " bytes)" << endl;
190190
return json;
191191
} catch (const exception &) { // caught by reference to base

‎benchmark/distinctuseridcompetition.cpp‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -266,11 +266,9 @@ int main(int argc, char *argv[]) {
266266
std::cerr << "warning: ignoring everything after " << argv[optind + 1]
267267
<< std::endl;
268268
}
269-
simdjson::padded_string p;
270-
try {
271-
simdjson::get_corpus(filename).swap(p);
272-
} catch (const std::exception &e) { // caught by reference to base
273-
std::cout << "Could not load the file " << filename << std::endl;
269+
auto [p, error] = simdjson::padded_string::load(filename);
270+
if (error) {
271+
std::cerr << "Could not load the file " << filename << std::endl;
274272
return EXIT_FAILURE;
275273
}
276274

‎benchmark/minifiercompetition.cpp‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -62,11 +62,9 @@ int main(int argc, char *argv[]) {
6262
exit(1);
6363
}
6464
const char *filename = argv[optind];
65-
simdjson::padded_string p;
66-
try {
67-
simdjson::get_corpus(filename).swap(p);
68-
} catch (const std::exception &e) { // caught by reference to base
69-
std::cout << "Could not load the file " << filename << std::endl;
65+
auto [p, error] = simdjson::padded_string::load(filename);
66+
if (error) {
67+
std::cerr << "Could not load the file " << filename << std::endl;
7068
return EXIT_FAILURE;
7169
}
7270
if (verbose) {
@@ -79,7 +77,7 @@ int main(int argc, char *argv[]) {
7977
std::cout << p.size() << " B ";
8078
std::cout << std::endl;
8179
}
82-
char *buffer = simdjson::allocate_padded_buffer(p.size() + 1);
80+
char *buffer = simdjson::internal::allocate_padded_buffer(p.size() + 1);
8381
memcpy(buffer, p.data(), p.size());
8482
buffer[p.size()] = '\0';
8583

@@ -122,7 +120,7 @@ int main(int argc, char *argv[]) {
122120
false, memcpy(buffer, p.data(), p.size()), repeat, volume,
123121
!just_data);
124122

125-
char *mini_buffer = simdjson::allocate_padded_buffer(p.size() + 1);
123+
char *mini_buffer = simdjson::internal::allocate_padded_buffer(p.size() + 1);
126124
size_t minisize = simdjson::json_minify((const uint8_t *)p.data(), p.size(),
127125
(uint8_t *)mini_buffer);
128126
mini_buffer[minisize] = '\0';

‎benchmark/parse_stream.cpp‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,8 @@ int main (int argc, char *argv[]){
2525
exit(1);
2626
}
2727
const char *filename = argv[1];
28-
simdjson::padded_string p;
29-
try {
30-
std::wclog << "loading " << filename << "\n" << std::endl;
31-
simdjson::get_corpus(filename).swap(p);
32-
} catch (const std::exception &) { // caught by reference to base
28+
auto [p, error] = simdjson::padded_string::load(filename);
29+
if (error) {
3330
std::cerr << "Could not load the file " << filename << std::endl;
3431
return EXIT_FAILURE;
3532
}

‎benchmark/parseandstatcompetition.cpp‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -268,11 +268,9 @@ int main(int argc, char *argv[]) {
268268
std::cerr << "warning: ignoring everything after " << argv[optind + 1]
269269
<< std::endl;
270270
}
271-
simdjson::padded_string p;
272-
try {
273-
simdjson::get_corpus(filename).swap(p);
274-
} catch (const std::exception &e) { // caught by reference to base
275-
std::cout << "Could not load the file " << filename << std::endl;
271+
auto [p, error] = simdjson::padded_string::load(filename);
272+
if (error) {
273+
std::cerr << "Could not load the file " << filename << std::endl;
276274
return EXIT_FAILURE;
277275
}
278276

‎benchmark/parsingcompetition.cpp‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,9 @@ size_t sum_line_lengths(char * data, size_t length) {
7373

7474

7575
bool bench(const char *filename, bool verbose, bool just_data, int repeat_multiplier) {
76-
simdjson::padded_string p;
77-
try {
78-
simdjson::get_corpus(filename).swap(p);
79-
} catch (const std::exception &e) { // caught by reference to base
80-
std::cout << "Could not load the file " << filename << std::endl;
76+
auto [p, err] = simdjson::padded_string::load(filename);
77+
if (err) {
78+
std::cerr << "Could not load the file " << filename << std::endl;
8179
return false;
8280
}
8381

0 commit comments

Comments
 (0)