libutil: begin porting serialization to generators
generators are a better basis for serializers than streaming into sinks as we do currently for many reasons, such as being usable as sources if one wishes to (without requiring an intermediate sink to serialize full data sets into memory, or boost coroutines to turn sinks into sources), composing more naturally (as one can just yield a sub-generator instead of being forced to wrap entire substreams into clunky functions or even more clunky custom types to implement operator<< on), allowing wrappers to transform data with clear ownership semantics (removing the need for explicit memory allocations and Source wrappers), and many other things Change-Id: I361d89ff556354f6930d9204f55117565f2f7f20
This commit is contained in:
eldritch horrors
parent
c65f5dd18e
commit
5eec6418de
5 changed files with 175 additions and 91 deletions
|
|
@ -160,8 +160,7 @@ struct TunnelSink : Sink
|
|||
TunnelSink(Sink & to) : to(to) { }
|
||||
void operator () (std::string_view data)
|
||||
{
|
||||
to << STDERR_WRITE;
|
||||
writeString(data, to);
|
||||
to << STDERR_WRITE << data;
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -897,7 +897,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
|
|||
if (!source) throw Error("no source");
|
||||
size_t len = readNum<size_t>(from);
|
||||
auto buf = std::make_unique<char[]>(len);
|
||||
writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to);
|
||||
to << std::string_view((const char *) buf.get(), source->read(buf.get(), len));
|
||||
to.flush();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -330,55 +330,40 @@ void writePadding(size_t len, Sink & sink)
|
|||
}
|
||||
|
||||
|
||||
void writeString(std::string_view data, Sink & sink)
|
||||
WireFormatGenerator SerializingTransform::operator()(std::string_view s)
|
||||
{
|
||||
sink << data.size();
|
||||
sink(data);
|
||||
writePadding(data.size(), sink);
|
||||
co_yield s.size();
|
||||
co_yield Bytes(s.begin(), s.size());
|
||||
co_yield SerializingTransform::padding(s.size());
|
||||
}
|
||||
|
||||
|
||||
Sink & operator << (Sink & sink, std::string_view s)
|
||||
WireFormatGenerator SerializingTransform::operator()(const Strings & ss)
|
||||
{
|
||||
writeString(s, sink);
|
||||
return sink;
|
||||
co_yield ss.size();
|
||||
for (const auto & s : ss)
|
||||
co_yield std::string_view(s);
|
||||
}
|
||||
|
||||
|
||||
template<class T> void writeStrings(const T & ss, Sink & sink)
|
||||
WireFormatGenerator SerializingTransform::operator()(const StringSet & ss)
|
||||
{
|
||||
sink << ss.size();
|
||||
for (auto & i : ss)
|
||||
sink << i;
|
||||
co_yield ss.size();
|
||||
for (const auto & s : ss)
|
||||
co_yield std::string_view(s);
|
||||
}
|
||||
|
||||
Sink & operator << (Sink & sink, const Strings & s)
|
||||
{
|
||||
writeStrings(s, sink);
|
||||
return sink;
|
||||
}
|
||||
|
||||
Sink & operator << (Sink & sink, const StringSet & s)
|
||||
{
|
||||
writeStrings(s, sink);
|
||||
return sink;
|
||||
}
|
||||
|
||||
Sink & operator << (Sink & sink, const Error & ex)
|
||||
WireFormatGenerator SerializingTransform::operator()(const Error & ex)
|
||||
{
|
||||
auto & info = ex.info();
|
||||
sink
|
||||
<< "Error"
|
||||
<< info.level
|
||||
<< "Error" // removed
|
||||
<< info.msg.str()
|
||||
<< 0 // FIXME: info.errPos
|
||||
<< info.traces.size();
|
||||
co_yield "Error";
|
||||
co_yield info.level;
|
||||
co_yield "Error"; // removed
|
||||
co_yield info.msg.str();
|
||||
co_yield 0; // FIXME: info.errPos
|
||||
co_yield info.traces.size();
|
||||
for (auto & trace : info.traces) {
|
||||
sink << 0; // FIXME: trace.pos
|
||||
sink << trace.hint.str();
|
||||
co_yield 0; // FIXME: trace.pos
|
||||
co_yield trace.hint.str();
|
||||
}
|
||||
return sink;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -350,33 +350,82 @@ inline Sink & operator<<(Sink & sink, Generator<Bytes> && g)
|
|||
return sink;
|
||||
}
|
||||
|
||||
void writePadding(size_t len, Sink & sink);
|
||||
void writeString(std::string_view s, Sink & sink);
|
||||
struct SerializingTransform;
|
||||
using WireFormatGenerator = Generator<Bytes, SerializingTransform>;
|
||||
|
||||
inline Sink & operator << (Sink & sink, uint64_t n)
|
||||
struct SerializingTransform
|
||||
{
|
||||
unsigned char buf[8];
|
||||
buf[0] = n & 0xff;
|
||||
buf[1] = (n >> 8) & 0xff;
|
||||
buf[2] = (n >> 16) & 0xff;
|
||||
buf[3] = (n >> 24) & 0xff;
|
||||
buf[4] = (n >> 32) & 0xff;
|
||||
buf[5] = (n >> 40) & 0xff;
|
||||
buf[6] = (n >> 48) & 0xff;
|
||||
buf[7] = (unsigned char) (n >> 56) & 0xff;
|
||||
sink({(char *) buf, sizeof(buf)});
|
||||
return sink;
|
||||
std::array<unsigned char, 8> buf;
|
||||
|
||||
Bytes operator()(uint64_t n)
|
||||
{
|
||||
buf[0] = n & 0xff;
|
||||
buf[1] = (n >> 8) & 0xff;
|
||||
buf[2] = (n >> 16) & 0xff;
|
||||
buf[3] = (n >> 24) & 0xff;
|
||||
buf[4] = (n >> 32) & 0xff;
|
||||
buf[5] = (n >> 40) & 0xff;
|
||||
buf[6] = (n >> 48) & 0xff;
|
||||
buf[7] = (unsigned char) (n >> 56) & 0xff;
|
||||
return {reinterpret_cast<const char *>(buf.begin()), 8};
|
||||
}
|
||||
|
||||
static Bytes padding(size_t unpadded)
|
||||
{
|
||||
return Bytes("\0\0\0\0\0\0\0", unpadded % 8 ? 8 - unpadded % 8 : 0);
|
||||
}
|
||||
|
||||
// opt in to generator chaining. without this co_yielding
|
||||
// another generator of any type will cause a type error.
|
||||
auto operator()(Generator<Bytes> && g)
|
||||
{
|
||||
return std::move(g);
|
||||
}
|
||||
|
||||
// only choose this for *exactly* char spans, do not allow implicit
|
||||
// conversions. this would cause ambiguities with strings literals,
|
||||
// and resolving those with more string-like overloads needs a lot.
|
||||
template<typename Span>
|
||||
requires std::same_as<Span, std::span<char>> || std::same_as<Span, std::span<const char>>
|
||||
Bytes operator()(Span s)
|
||||
{
|
||||
return s;
|
||||
}
|
||||
WireFormatGenerator operator()(std::string_view s);
|
||||
WireFormatGenerator operator()(const Strings & s);
|
||||
WireFormatGenerator operator()(const StringSet & s);
|
||||
WireFormatGenerator operator()(const Error & s);
|
||||
};
|
||||
|
||||
void writePadding(size_t len, Sink & sink);
|
||||
|
||||
inline Sink & operator<<(Sink & sink, uint64_t u)
|
||||
{
|
||||
return sink << [&]() -> WireFormatGenerator { co_yield u; }();
|
||||
}
|
||||
|
||||
Sink & operator << (Sink & in, const Error & ex);
|
||||
Sink & operator << (Sink & sink, std::string_view s);
|
||||
Sink & operator << (Sink & sink, const Strings & s);
|
||||
Sink & operator << (Sink & sink, const StringSet & s);
|
||||
inline Sink & operator<<(Sink & sink, std::string_view s)
|
||||
{
|
||||
return sink << [&]() -> WireFormatGenerator { co_yield s; }();
|
||||
}
|
||||
|
||||
inline Sink & operator<<(Sink & sink, const Strings & s)
|
||||
{
|
||||
return sink << [&]() -> WireFormatGenerator { co_yield s; }();
|
||||
}
|
||||
|
||||
inline Sink & operator<<(Sink & sink, const StringSet & s)
|
||||
{
|
||||
return sink << [&]() -> WireFormatGenerator { co_yield s; }();
|
||||
}
|
||||
|
||||
inline Sink & operator<<(Sink & sink, const Error & ex)
|
||||
{
|
||||
return sink << [&]() -> WireFormatGenerator { co_yield ex; }();
|
||||
}
|
||||
|
||||
MakeError(SerialisationError, Error);
|
||||
|
||||
|
||||
template<typename T>
|
||||
T readNum(Source & source)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -2,30 +2,47 @@
|
|||
#include "error.hh"
|
||||
#include "fmt.hh"
|
||||
#include "pos-table.hh"
|
||||
#include "generator.hh"
|
||||
#include "ref.hh"
|
||||
#include "types.hh"
|
||||
|
||||
#include <concepts>
|
||||
#include <cstdint>
|
||||
#include <initializer_list>
|
||||
#include <limits.h>
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <numeric>
|
||||
#include <stdexcept>
|
||||
#include <string_view>
|
||||
#include <type_traits>
|
||||
|
||||
namespace nix {
|
||||
|
||||
TEST(Sink, uint64_t)
|
||||
// don't deduce the type of `val` for added insurance.
|
||||
template<typename T>
|
||||
static std::string toWire(const std::type_identity_t<T> & val)
|
||||
{
|
||||
StringSink s;
|
||||
s << 42;
|
||||
ASSERT_EQ(s.s, std::string({42, 0, 0, 0, 0, 0, 0, 0}));
|
||||
std::string result;
|
||||
auto g = [] (const auto & val) -> WireFormatGenerator { co_yield val; }(val);
|
||||
while (auto buffer = g.next()) {
|
||||
result.append(buffer->data(), buffer->size());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
TEST(Sink, string_view)
|
||||
TEST(WireFormatGenerator, uint64_t)
|
||||
{
|
||||
StringSink s;
|
||||
s << "";
|
||||
auto s = toWire<uint64_t>(42);
|
||||
ASSERT_EQ(s, std::string({42, 0, 0, 0, 0, 0, 0, 0}));
|
||||
}
|
||||
|
||||
TEST(WireFormatGenerator, string_view)
|
||||
{
|
||||
auto s = toWire<std::string_view>("");
|
||||
// clang-format off
|
||||
ASSERT_EQ(
|
||||
s.s,
|
||||
s,
|
||||
std::string({
|
||||
// length
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
|
|
@ -34,11 +51,10 @@ TEST(Sink, string_view)
|
|||
);
|
||||
// clang-format on
|
||||
|
||||
s = {};
|
||||
s << "test";
|
||||
s = toWire<std::string_view>("test");
|
||||
// clang-format off
|
||||
ASSERT_EQ(
|
||||
s.s,
|
||||
s,
|
||||
std::string({
|
||||
// length
|
||||
4, 0, 0, 0, 0, 0, 0, 0,
|
||||
|
|
@ -50,11 +66,10 @@ TEST(Sink, string_view)
|
|||
);
|
||||
// clang-format on
|
||||
|
||||
s = {};
|
||||
s << "longer string";
|
||||
s = toWire<std::string_view>("longer string");
|
||||
// clang-format off
|
||||
ASSERT_EQ(
|
||||
s.s,
|
||||
s,
|
||||
std::string({
|
||||
// length
|
||||
13, 0, 0, 0, 0, 0, 0, 0,
|
||||
|
|
@ -67,13 +82,12 @@ TEST(Sink, string_view)
|
|||
// clang-format on
|
||||
}
|
||||
|
||||
TEST(Sink, StringSet)
|
||||
TEST(WireFormatGenerator, StringSet)
|
||||
{
|
||||
StringSink s;
|
||||
s << StringSet{};
|
||||
auto s = toWire<StringSet>({});
|
||||
// clang-format off
|
||||
ASSERT_EQ(
|
||||
s.s,
|
||||
s,
|
||||
std::string({
|
||||
// length
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
|
|
@ -82,11 +96,10 @@ TEST(Sink, StringSet)
|
|||
);
|
||||
// clang-format on
|
||||
|
||||
s = {};
|
||||
s << StringSet{"a", ""};
|
||||
s = toWire<StringSet>({"a", ""});
|
||||
// clang-format off
|
||||
ASSERT_EQ(
|
||||
s.s,
|
||||
s,
|
||||
std::string({
|
||||
// length
|
||||
2, 0, 0, 0, 0, 0, 0, 0,
|
||||
|
|
@ -99,13 +112,12 @@ TEST(Sink, StringSet)
|
|||
// clang-format on
|
||||
}
|
||||
|
||||
TEST(Sink, Strings)
|
||||
TEST(WireFormatGenerator, Strings)
|
||||
{
|
||||
StringSink s;
|
||||
s << Strings{};
|
||||
auto s = toWire<Strings>({});
|
||||
// clang-format off
|
||||
ASSERT_EQ(
|
||||
s.s,
|
||||
s,
|
||||
std::string({
|
||||
// length
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
|
|
@ -114,11 +126,10 @@ TEST(Sink, Strings)
|
|||
);
|
||||
// clang-format on
|
||||
|
||||
s = {};
|
||||
s << Strings{"a", ""};
|
||||
s = toWire<Strings>({"a", ""});
|
||||
// clang-format off
|
||||
ASSERT_EQ(
|
||||
s.s,
|
||||
s,
|
||||
std::string({
|
||||
// length
|
||||
2, 0, 0, 0, 0, 0, 0, 0,
|
||||
|
|
@ -131,23 +142,22 @@ TEST(Sink, Strings)
|
|||
// clang-format on
|
||||
}
|
||||
|
||||
TEST(Sink, Error)
|
||||
TEST(WireFormatGenerator, Error)
|
||||
{
|
||||
PosTable pt;
|
||||
auto o = pt.addOrigin(Pos::String{make_ref<std::string>("test")}, 4);
|
||||
|
||||
StringSink s;
|
||||
s << Error{ErrorInfo{
|
||||
auto s = toWire<Error>(Error{ErrorInfo{
|
||||
.level = lvlInfo,
|
||||
.msg = HintFmt("foo"),
|
||||
.pos = pt[pt.add(o, 1)],
|
||||
.traces = {{.pos = pt[pt.add(o, 2)], .hint = HintFmt("b %1%", "foo")}},
|
||||
}};
|
||||
}});
|
||||
// NOTE position of the error and all traces are ignored
|
||||
// by the wire format
|
||||
// clang-format off
|
||||
ASSERT_EQ(
|
||||
s.s,
|
||||
s,
|
||||
std::string({
|
||||
5, 0, 0, 0, 0, 0, 0, 0, 'E', 'r', 'r', 'o', 'r', 0, 0, 0,
|
||||
3, 0, 0, 0, 0, 0, 0, 0,
|
||||
|
|
@ -163,4 +173,45 @@ TEST(Sink, Error)
|
|||
// clang-format on
|
||||
}
|
||||
|
||||
TEST(WireFormatGenerator, exampleMessage)
|
||||
{
|
||||
auto gen = []() -> WireFormatGenerator {
|
||||
std::set<std::string> foo{"a", "longer string", ""};
|
||||
co_yield 42;
|
||||
co_yield foo;
|
||||
co_yield std::string_view("test");
|
||||
co_yield true;
|
||||
}();
|
||||
|
||||
std::vector<char> full;
|
||||
while (auto s = gen.next()) {
|
||||
full.insert(full.end(), s->begin(), s->end());
|
||||
}
|
||||
|
||||
ASSERT_EQ(
|
||||
full,
|
||||
(std::vector<char>{
|
||||
// clang-format off
|
||||
// 42
|
||||
42, 0, 0, 0, 0, 0, 0, 0,
|
||||
// foo
|
||||
3, 0, 0, 0, 0, 0, 0, 0,
|
||||
/// ""
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
/// a
|
||||
1, 0, 0, 0, 0, 0, 0, 0,
|
||||
'a', 0, 0, 0, 0, 0, 0, 0,
|
||||
/// longer string
|
||||
13, 0, 0, 0, 0, 0, 0, 0,
|
||||
'l', 'o', 'n', 'g', 'e', 'r', ' ', 's', 't', 'r', 'i', 'n', 'g', 0, 0, 0,
|
||||
// foo done
|
||||
// test
|
||||
4, 0, 0, 0, 0, 0, 0, 0,
|
||||
't', 'e', 's', 't', 0, 0, 0, 0,
|
||||
// true
|
||||
1, 0, 0, 0, 0, 0, 0, 0,
|
||||
//clang-format on
|
||||
}));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue