38#include <google/protobuf/descriptor.h>
43#include "dynamic_protobuf_manager.h"
45#include "field_codec.h"
46#include "field_codec_fixed.h"
49#include "codecs2/field_codec_default_message.h"
50#include "codecs3/field_codec_default_message.h"
52#include "dccl/version.h"
53#include "field_codec_manager.h"
70 Codec(std::string dccl_id_codec_name = default_id_codec_name(),
71 const std::string& library_path =
"");
79 template <
class IDFieldCodec,
80 typename std::enable_if<std::is_base_of<FieldCodecBase, IDFieldCodec>::value,
82 Codec(
const std::string& dccl_id_codec_name,
const IDFieldCodec& dccl_id_codec)
83 : id_codec_(dccl_id_codec_name)
86 manager_.
add<IDFieldCodec>(dccl_id_codec_name);
121 template <
typename ProtobufMessage> std::size_t
load()
123 return load(ProtobufMessage::descriptor());
129 template <
typename ProtobufMessage>
void unload() {
unload(ProtobufMessage::descriptor()); }
131 void unload_all() { id2desc_.clear(); }
140 std::size_t
load(
const google::protobuf::Descriptor* desc,
int user_id = -1);
146 void unload(
const google::protobuf::Descriptor* desc);
152 void unload(
size_t dccl_id);
156 std::string get_id_codec() {
return id_codec_; }
164 const std::set<int32>& do_not_encrypt_ids = std::set<int32>());
167 const std::set<unsigned>& do_not_encrypt_ids)
169 std::set<int32> s_ids{do_not_encrypt_ids.begin(), do_not_encrypt_ids.end()};
196 template <
typename ProtobufMessage>
197 void info(std::ostream* os =
nullptr,
int user_id = -1)
const
199 info(ProtobufMessage::descriptor(), os, user_id);
207 void info(
const google::protobuf::Descriptor* desc, std::ostream* os =
nullptr,
208 int user_id = -1)
const;
213 void info_all(std::ostream* os =
nullptr)
const;
218 template <
typename ProtobufMessage>
int32 id()
const
220 return id(ProtobufMessage::descriptor());
242 int32 id(
const std::string& bytes)
const;
245 template <
typename CharIterator>
int32 id(CharIterator begin, CharIterator end)
const;
248 int32 id(
const google::protobuf::Descriptor* desc)
const
250 if (desc->options().GetExtension(dccl::msg).omit_id())
251 throw(
Exception(
"Cannot call id(...) on message with omit_id == true"));
253 dccl::uint32 hardcoded_id = desc->options().GetExtension(dccl::msg).id();
257 id_codec()->field_encode(&id_bits, hardcoded_id,
nullptr);
263 const std::map<int32, const google::protobuf::Descriptor*>&
loaded()
const {
return id2desc_; }
280 void encode(std::string* bytes,
const google::protobuf::Message& msg,
bool header_only =
false,
293 size_t encode(
char* bytes,
size_t max_len,
const google::protobuf::Message& msg,
294 bool header_only =
false,
int user_id = -1);
304 template <
typename CharIterator,
typename ProtobufMessage>
305 CharIterator
decode(CharIterator begin, CharIterator end, ProtobufMessage* msg,
306 bool header_only =
false);
314 template <
typename ProtobufMessage>
315 void decode(
const std::string& bytes, ProtobufMessage* msg,
bool header_only =
false)
317 decode(bytes.begin(), bytes.end(), msg, header_only);
325 template <
typename ProtobufMessage>
void decode(std::string* bytes, ProtobufMessage* msg)
328 unsigned last_size =
size(*msg);
329 bytes->erase(0, last_size);
339 template <
typename GoogleProtobufMessagePo
inter>
340 GoogleProtobufMessagePointer
decode(
const std::string& bytes,
bool header_only =
false);
348 template <
typename GoogleProtobufMessagePo
inter>
349 GoogleProtobufMessagePointer
decode(std::string* bytes);
357 unsigned size(
const google::protobuf::Message& msg,
int user_id = -1);
363 template <
typename ProtobufMessage>
unsigned max_size()
365 return max_size(ProtobufMessage::descriptor());
369 unsigned max_size(
const google::protobuf::Descriptor* desc)
const;
375 template <
typename ProtobufMessage>
unsigned min_size()
377 return min_size(ProtobufMessage::descriptor());
381 unsigned min_size(
const google::protobuf::Descriptor* desc)
const;
385 static std::string default_id_codec_name() {
return "dccl.default.id"; }
387 static std::string default_codec_name(
int version = 2)
389 return "dccl.default" + std::to_string(version);
392 FieldCodecManagerLocal& manager() {
return manager_; }
395 void encode_internal(
const google::protobuf::Message& msg,
bool header_only,
396 Bitset& header_bits, Bitset& body_bits,
int user_id);
397 std::string get_all_error_fields_in_message(
const google::protobuf::Message& msg,
400 void encrypt(std::string* s,
const std::string& nonce);
401 void decrypt(std::string* s,
const std::string& nonce);
403 void set_default_codecs();
405 std::shared_ptr<FieldCodecBase> id_codec()
const
407 return manager_.
find(google::protobuf::FieldDescriptor::TYPE_UINT32, DCCL_VERSION_MAJOR,
411 int32 id_internal(
const google::protobuf::Descriptor* desc,
int user_id)
414 if (desc->options().GetExtension(dccl::msg).omit_id() && !desc2placeholder_id_.count(desc))
415 desc2placeholder_id_.insert(std::make_pair(desc, omit_id_placeholder_id_--));
417 return id_internal_const(desc, user_id);
420 int32 id_internal_const(
const google::protobuf::Descriptor* desc,
int user_id)
const
422 if (desc->options().GetExtension(dccl::msg).omit_id())
424 if (desc2placeholder_id_.count(desc))
425 return desc2placeholder_id_.find(desc)->second;
427 throw(Exception(
"Message " + desc->full_name() +
428 " has omit_id == true but has not been loaded, so id_internal() "
429 "const cannot be called"));
433 return (user_id < 0) ?
id(desc) : user_id;
439 std::string crypto_key_;
445 unsigned console_width_{60};
448 std::set<int32> skip_crypto_ids_;
451 std::map<int32, const google::protobuf::Descriptor*> id2desc_;
452 std::string id_codec_;
454 std::vector<void*> dl_handles_;
456 std::string build_guard_for_console_output(std::string& base,
char guard_char)
const;
458 FieldCodecManagerLocal manager_;
461 int32 omit_id_placeholder_id_{-1};
463 std::map<const google::protobuf::Descriptor*, int32> desc2placeholder_id_;
466inline std::ostream& operator<<(std::ostream& os,
const Codec& codec)
473template <
typename GoogleProtobufMessagePo
inter>
479 if (!id2desc_.count(this_id))
480 throw(
Exception(
"Message id " + std::to_string(this_id) +
481 " has not been loaded. Call load() before decoding this type."));
484 auto msg = dccl::DynamicProtobufManager::new_protobuf_message<GoogleProtobufMessagePointer>(
485 id2desc_.find(this_id)->second);
486 decode(bytes, &(*msg), header_only);
490template <
typename GoogleProtobufMessagePo
inter>
493 int32 this_id = id(*bytes);
495 if (!id2desc_.count(this_id))
496 throw(
Exception(
"Message id " + std::to_string(this_id) +
497 " has not been loaded. Call load() before decoding this type."));
499 GoogleProtobufMessagePointer msg =
500 dccl::DynamicProtobufManager::new_protobuf_message<GoogleProtobufMessagePointer>(
501 id2desc_.find(this_id)->second);
502 std::string::iterator new_begin = decode(bytes->begin(), bytes->end(), &(*msg));
503 bytes->erase(bytes->begin(), new_begin);
507template <
typename CharIterator>
512 unsigned id_min_size = 0, id_max_size = 0;
513 id_codec()->field_min_size(&id_min_size,
nullptr);
514 id_codec()->field_max_size(&id_max_size,
nullptr);
518 int incr = std::min<size_t>(
519 static_cast<size_t>(std::distance(begin, end)),
520 static_cast<size_t>(std::ceil(
static_cast<double>(id_max_size) / BITS_IN_BYTE)));
523 Bitset these_bits(&fixed_header_bits);
524 these_bits.get_more_bits(id_min_size);
526 dccl::any return_value;
527 id_codec()->field_decode(&these_bits, &return_value,
nullptr);
528 return dccl::any_cast<uint32>(return_value);
532 throw(Exception(
"Failed to decoded id from bytes passed (hex: " + hex_encode(begin, end) +
537template <
typename CharIterator,
typename ProtobufMessage>
543 const google::protobuf::Descriptor* desc = msg->GetDescriptor();
544 int32 expected_id = id_internal(desc, -1);
547 if (!desc->options().GetExtension(dccl::msg).omit_id())
549 received_id = id(begin, end);
551 if (!id2desc_.count(received_id))
552 throw(
Exception(
"Message id " + std::to_string(received_id) +
553 " has not been loaded. Call load() before decoding this type."));
555 if (expected_id != received_id)
556 throw(
Exception(
"Received message with id " + std::to_string(received_id) +
" (" +
557 id2desc_.at(received_id)->full_name() +
558 ") but decode was called with message of id " +
559 std::to_string(expected_id) +
" (" + desc->full_name() +
560 "). Ensure dccl::Codec::decode is called with the correct Protobuf "
561 "message or use the dynamic overloads of decode."));
564 dlog.
is(logger::DEBUG1, logger::DECODE) &&
565 dlog <<
"Began decoding message of id: " << received_id << std::endl;
567 dlog.
is(logger::DEBUG1, logger::DECODE) && dlog <<
"Type name: " << desc->full_name()
570 std::shared_ptr<FieldCodecBase> codec = manager_.find(desc);
571 std::shared_ptr<internal::FromProtoCppTypeBase> helper = manager_.type_helper().find(desc);
573 CharIterator actual_end = end;
576 unsigned head_size_bits;
577 unsigned body_size_bits;
578 codec->base_max_size(&head_size_bits, desc, HEAD);
579 codec->base_max_size(&body_size_bits, desc, BODY);
580 unsigned id_size = 0;
581 if (!desc->options().GetExtension(dccl::msg).omit_id())
582 id_codec()->field_size(&id_size,
static_cast<uint32>(received_id),
nullptr);
583 head_size_bits += id_size;
585 unsigned head_size_bytes = ceil_bits2bytes(head_size_bits);
586 unsigned body_size_bytes = ceil_bits2bytes(body_size_bits);
588 dlog.
is(logger::DEBUG2, logger::DECODE) &&
589 dlog <<
"Head bytes (bits): " << head_size_bytes <<
"(" << head_size_bits
590 <<
"), max body bytes (bits): " << body_size_bytes <<
"(" << body_size_bits
593 CharIterator head_bytes_end = begin + head_size_bytes;
594 dlog.
is(logger::DEBUG3, logger::DECODE) &&
595 dlog <<
"Unencrypted Head (hex): " <<
hex_encode(begin, head_bytes_end)
600 dlog.
is(logger::DEBUG3, logger::DECODE) &&
601 dlog <<
"Unencrypted Head (bin): " << head_bits << std::endl;
604 head_bits >>= id_size;
606 dlog.
is(logger::DEBUG3, logger::DECODE) &&
607 dlog <<
"Unencrypted Head after ID bits removal (bin): " << head_bits << std::endl;
610 manager_.codec_data().message_data_);
611 msg_stack.push(msg->GetDescriptor());
613 codec->base_decode(&head_bits, msg, HEAD);
614 dlog.
is(logger::DEBUG2, logger::DECODE) &&
615 dlog <<
"after header decode, message is: " << *msg << std::endl;
619 dlog.
is(logger::DEBUG2, logger::DECODE) &&
620 dlog <<
"as requested, skipping decrypting and decoding body." << std::endl;
621 actual_end = head_bytes_end;
625 dlog.
is(logger::DEBUG3, logger::DECODE) &&
626 dlog <<
"Encrypted Body (hex): " <<
hex_encode(head_bytes_end, end)
630 if (!crypto_key_.empty() && !skip_crypto_ids_.count(received_id))
632 std::string head_bytes(begin, head_bytes_end);
633 std::string body_bytes(head_bytes_end, end);
634 decrypt(&body_bytes, head_bytes);
635 dlog.
is(logger::DEBUG3, logger::DECODE) &&
636 dlog <<
"Unencrypted Body (hex): " <<
hex_encode(body_bytes) << std::endl;
641 dlog.
is(logger::DEBUG3, logger::DECODE) &&
642 dlog <<
"Unencrypted Body (hex): " <<
hex_encode(head_bytes_end, end)
647 dlog.
is(logger::DEBUG3, logger::DECODE) &&
648 dlog <<
"Unencrypted Body (bin): " << body_bits << std::endl;
650 codec->base_decode(&body_bits, msg, BODY);
651 dlog.
is(logger::DEBUG2, logger::DECODE) &&
652 dlog <<
"after header & body decode, message is: " << *msg << std::endl;
654 actual_end = end - body_bits.size() / BITS_IN_BYTE;
659 throw(
Exception(
"Failed to find (dccl.msg).codec `" +
660 desc->options().GetExtension(dccl::msg).codec() +
"`"),
664 dlog.
is(logger::DEBUG1, logger::DECODE) &&
665 dlog <<
"Successfully decoded message of type: " << desc->full_name() << std::endl;
668 catch (std::exception& e)
670 std::stringstream ss;
672 ss <<
"Message " <<
hex_encode(begin, end) <<
" failed to decode. Reason: " << e.what()
675 dlog.
is(logger::DEBUG1, logger::DECODE) && dlog << ss.str() << std::endl;
A variable size container of bits (subclassed from std::deque<bool>) with an optional hierarchy....
std::string to_byte_string()
Returns the value of the Bitset to a byte string, where each character represents 8 bits of the Bitse...
void from_byte_stream(CharIterator begin, CharIterator end)
Sets the value of the Bitset to the contents of a byte string, where each character represents 8 bits...
The Dynamic CCL enCODer/DECoder. This is the main class you will use to load, encode and decode DCCL ...
int32 id(const google::protobuf::Descriptor *desc) const
Provides the DCCL ID given a DCCL type.
void info(std::ostream *os=nullptr, int user_id=-1) const
Writes a human readable summary (including field sizes) of the provided DCCL type to the stream provi...
void set_strict(bool mode)
Set "strict" mode where a dccl::OutOfRangeException will be thrown for encode if the value(s) provide...
void info_all(std::ostream *os=nullptr) const
Writes a human readable summary (including field sizes) of all the loaded (validated) DCCL types.
const std::map< int32, const google::protobuf::Descriptor * > & loaded() const
Provides a map of all loaded DCCL IDs to the equivalent Protobuf descriptor.
void set_id_codec(const std::string &id_codec_name)
Set a different ID codec name (note that is calls unload_all() so all messages must be reloaded)
unsigned min_size()
Provides the encoded minimum size (in bytes) of msg.
void decode(std::string *bytes, ProtobufMessage *msg)
Decode a DCCL message when the type is known at compile time.
int32 id(CharIterator begin, CharIterator end) const
Get the DCCL ID of an unknown encoded DCCL message (Iterator overload).
int32 id() const
Gives the DCCL id (defined by the custom message option extension "(dccl.msg).id" in the ....
void unload_library(void *dl_handle)
Remove codecs and/or unload messages present in the given shared library handle.
unsigned max_size()
Provides the encoded maximum size (in bytes) of msg.
void encode(std::string *bytes, const google::protobuf::Message &msg, bool header_only=false, int user_id=-1)
Encodes a DCCL message.
void set_console_width(unsigned num_chars)
Set the number of characters used in programmatic generation of console outputs.
void unload()
Unload a given message.
void set_crypto_passphrase(const std::string &passphrase, const std::set< int32 > &do_not_encrypt_ids=std::set< int32 >())
Set a passphrase to be used when encoded messages to encrypt them and to decrypt messages after decod...
virtual ~Codec()
Destructor.
CharIterator decode(CharIterator begin, CharIterator end, ProtobufMessage *msg, bool header_only=false)
Decode a DCCL message when the type is known at compile time.
Codec(const std::string &dccl_id_codec_name, const IDFieldCodec &dccl_id_codec)
Instantiate a Codec with a non-default identifier field codec (loaded directly).
void load_library(void *dl_handle)
Add codecs and/or load messages present in the given shared library handle.
unsigned size(const google::protobuf::Message &msg, int user_id=-1)
Provides the encoded size (in bytes) of msg. This is useful if you need to know the size of a message...
std::size_t load()
All messages must be explicited loaded and validated (size checks, option extensions checks,...
void decode(const std::string &bytes, ProtobufMessage *msg, bool header_only=false)
Decode a DCCL message when the type is known at compile time.
Exception class for DCCL.
std::enable_if< std::is_base_of< google::protobuf::Message, typenameCodec::wire_type >::value &&!std::is_same< google::protobuf::Message, typenameCodec::wire_type >::value, void >::type add(const std::string &name)
Add a new field codec (used for codecs operating on statically generated Protobuf messages,...
std::shared_ptr< FieldCodecBase > find(const google::protobuf::FieldDescriptor *field, int codec_version, bool has_codec_group, const std::string &codec_group) const
Find the codec for a given field. For embedded messages, prefers (dccl.field).codec (inside field) ov...
bool is(logger::Verbosity verbosity, logger::Group group=logger::GENERAL)
Indicates the verbosity of the Logger until the next std::flush or std::endl. The boolean return is u...
Dynamic Compact Control Language namespace.
google::protobuf::int32 int32
a signed 32 bit integer
google::protobuf::uint32 uint32
an unsigned 32 bit integer
void hex_encode(CharIterator begin, CharIterator end, std::string *out, bool upper_case=false)
Encodes a (little-endian) hexadecimal string from a byte string. Index 0 of begin is written to index...