Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/bjdata.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,9 @@ fields. Every schema field type is understood: fixed-width scalars, the `T` bool
padded `S`/`H` text, dictionaries, offset-table text, nested records and fixed runs. A member of
the type that the schema has no field for keeps its value unless the type insists on it; a field
the type has no member for is skipped. A table with more than one dimension is read flat, in
record order.
record order. A record must take a byte at least: a schema of null fields and text of no length,
whose records would take none, is refused as a strong type with no payload is, since a count of
such records is held to no bytes.

Writing goes the other way from the type: a range of two or more records of a described type
whose every member a schema can say — a number, a boolean, a character, text, a mapped
Expand Down
13 changes: 9 additions & 4 deletions include/serpent/bjdata/soa.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -251,11 +251,17 @@ inline std::uint32_t parse_type(cursor &source, schema &into, std::uint32_t inde
*
* Every field's offset is where it lies in a row-major record; a column-major table lays a
* top-level field's values out together, and the same offsets serve inside each of those.
*
* A record has to take a byte at least. One of null fields and empty text takes none, and a
* count of such records would be a count no bytes hold to - as a strong type with no payload
* would be, which is refused for the same reason.
*/
[[nodiscard]] inline schema parse(bjdata::detail::cursor &source) noexcept {
const std::byte *const opening = source.position - 1;
schema result;
result.record_bytes = detail::parse_fields(source, result, 0);
if (source.ok() && result.fields.empty()) source.fail(errc::invalid_dimensions);
if (source.ok() && result.record_bytes == 0) source.fail(errc::invalid_strong_type, opening);
if (!source.ok()) return result;
for (std::uint32_t index = 0; index < result.count(); index = result.fields[index].next) ++result.top_level;
return result;
Expand Down Expand Up @@ -529,10 +535,9 @@ inline void skip_structured(cursor &source, const container_prefix &info, bool b
soa::skip_payload(source, layout, info.count);
if (!source.ok()) return;

// Only a boolean, a dictionary index or an offset index can hold a byte no handle reads, and
// each takes a byte a record at least, so the payload just stepped over holds the count to
// the bytes there. A table with none of them is not walked at all: its records may take no
// bytes, and claim any number.
// Only a boolean, a dictionary index or an offset index can hold a byte no handle reads, so a
// table with none of them is not walked at all. Every record takes a byte at least, so the
// payload just stepped over has held the count to the bytes there either way.
const bool checked = std::ranges::any_of(layout, [](const soa::field &field) {
return field.kind == soa::field_kind::boolean || field.kind == soa::field_kind::dictionary || field.kind == soa::field_kind::offsets;
});
Expand Down
9 changes: 7 additions & 2 deletions include/serpent/bjdata/soa_write.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -56,20 +56,25 @@ consteval bool mappable() {
return column_of<T>() != column::none;
}

/** A described type every member of which a schema can say. */
/**
* A described type every member of which a schema can say, with one member at least: every
* column takes a byte a record, and a record of none would take none, which a table may not.
*/
template<typename T>
consteval bool is_table_record() {
if constexpr (!reflected_type<T>) {
return false;
} else {
bool every = true;
bool any = false;
template for (constexpr auto member : std::define_static_array(serpent::detail::members_including_bases<T>())) {
if constexpr (!serpent::detail::has_annotation<skip>(member)) {
using field = std::remove_cvref_t<typename [:std::meta::type_of(member):]>;
if constexpr (serpent::detail::member_is_projected<member>() || !mappable<field>()) every = false;
any = true;
}
}
return every;
return every && any;
}
}

Expand Down
39 changes: 28 additions & 11 deletions test/bjdata_container_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -257,18 +257,35 @@ void table_fields() {
check(!tried && checked.has_value() == false && tried.error().code() == checked.error().code(), what + ": and try_decode says why");
}

// Only a field with a byte to check costs anything a record, and each such field takes a byte,
// so a count the bytes cannot hold is refused before anything is walked; a table of records
// that take no bytes is not walked at all, whatever it claims.
std::string zero_width { "[${U\x01" "aZ}#L" };
std::string with_boolean { "[${U\x01" "aZU\x01" "bT}#L" };
for (int index = 0; index < 8; ++index) {
zero_width += static_cast<char>(index == 5 ? 1 : 0); // 2^40 records
with_boolean += static_cast<char>(index == 5 ? 1 : 0);
// Every record takes a byte at least, as every element of a typed container does: a schema
// whose fields take none - null fields, text of no length, records and runs of those - is
// refused where it opens, whatever the count, so no count can outrun the bytes there.
const auto count_of = [](std::string schema, std::uint64_t records) {
schema += "}#L";
for (int index = 0; index < 8; ++index) schema += static_cast<char>(records >> (8 * index));
return schema;
};
const std::uint64_t huge = std::uint64_t { 1 } << 40;
for (const auto &[what, schema] : std::array<std::pair<const char *, std::string>, 5> { {
{ "a null field", std::string { "[${U\x01" "aZ" } },
{ "text of no length", std::string { "[${U\x01" "aSU\x00", 9 } },
{ "a nested record of null fields", std::string { "[${U\x01" "r{U\x01" "aZU\x01" "bZ}" } },
{ "a run of null fields", std::string { "[${U\x01" "r[ZZ]" } },
{ "a column-major table of null fields", std::string { "{${U\x01" "aZ" } },
} }) {
for (const auto records : { std::uint64_t { 0 }, std::uint64_t { 3 }, huge }) {
const auto document = bytes_of(count_of(schema, records));
const auto checked = validate(document);
const std::string named = std::string { "a schema of " } + what + ", " + std::to_string(records) + " records";
check(!checked && checked.error().code() == errc::invalid_strong_type && checked.error().offset() == 2,
named + ": refused at the schema");
check(!decode<value>(document).has_value(), named + ": does not decode");
}
}
check(validate(bytes_of(zero_width)).has_value(), "a table of 2^40 records of no bytes is stepped over, not walked");
const auto unheld = validate(bytes_of(with_boolean));
check(!unheld && unheld.error().code() == errc::unexpected_end, "and 2^40 records of a byte each are refused by their length");
const auto beside = bytes_of(count_of("[${U\x01" "aZU\x01" "bT", 2) + "TF");
check(validate(beside).has_value() && decode<value>(beside).has_value(), "a null field beside one that takes a byte is a table");
const auto unheld = validate(bytes_of(count_of("[${U\x01" "aZU\x01" "bT", huge)));
check(!unheld && unheld.error().code() == errc::unexpected_end, "and 2^40 of its records are refused by their length");

// The block notation traces bytes, and stops at one that is not what the schema says.
const auto notation = block_notation(bytes_of("[${U\x01" "aT}#U\x03" "TFU"sv));
Expand Down
18 changes: 18 additions & 0 deletions test/bjdata_table_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,15 @@ struct [[= serpent::serializable {}]] demanding {
int absent = 0; ///< no such field, and not defaulted
};

/** Types a table could hold no byte of a record of: nothing to say, or nothing said. */
struct [[= serpent::serializable {}]] nothing {};
struct [[= serpent::serializable {}]] unsaid {
[[= serpent::skip {}]] int never = 0;
};
struct [[= serpent::serializable {}]] holding_nothing {
nothing inner;
};

std::vector<std::byte> read_file(const std::filesystem::path &path) {
std::ifstream in { path, std::ios::binary };
std::stringstream buffer;
Expand Down Expand Up @@ -216,6 +225,15 @@ int main() {
// reads back as themselves and as the tree dart-bjdata's table is.
const std::vector<record> forty = *by_row;
check(bjdata::encode(forty)[1] != std::byte { '$' }, "a type with an optional member writes an array of objects");
// A record must take a byte, so a type whose records would take none is no table record.
const auto untabled = [](const auto &records, const char *what) {
const auto bytes = bjdata::encode(records);
check(bytes.size() > 1 && bytes[1] != std::byte { '$' }, what);
check(bjdata::validate(bytes).has_value(), what);
};
untabled(std::vector<nothing>(3), "a type with no members writes an array of objects");
untabled(std::vector<unsaid>(3), "a type whose every member is skipped writes one too");
untabled(std::vector<holding_nothing>(3), "and a type holding only such a record");
std::vector<table_row> rows_out;
for (const auto &row : forty) rows_out.push_back({ row.at, row.pos, row.v, row.station, row.flag, row.note });
const auto written = bjdata::encode(rows_out);
Expand Down
Loading