38 #if CRYPTOPP_PATH_USES_PLUS_SIGN
39 #include <crypto++/aes.h>
40 #include <crypto++/filters.h>
41 #include <crypto++/modes.h>
42 #include <crypto++/sha.h>
44 #include <cryptopp/aes.h>
45 #include <cryptopp/filters.h>
46 #include <cryptopp/modes.h>
47 #include <cryptopp/sha.h>
48 #endif // CRYPTOPP_PATH_USES_PLUS_SIGN
49 #endif // HAS_CRYPTOPP
51 #include "codecs2/field_codec_default.h"
52 #include "codecs3/field_codec_default.h"
53 #include "codecs3/field_codec_presence.h"
54 #include "codecs3/field_codec_var_bytes.h"
55 #include "codecs4/field_codec_default.h"
56 #include "codecs4/field_codec_default_message.h"
57 #include "field_codec_id.h"
59 #include "option_extensions.pb.h"
64 using namespace dccl::logger;
66 using google::protobuf::Descriptor;
67 using google::protobuf::FieldDescriptor;
68 using google::protobuf::Reflection;
75 : id_codec_(std::move(dccl_id_codec_name))
80 if (!library_path.empty())
88 for (
auto& dl_handle : dl_handles_)
90 unload_library(dl_handle);
95 void dccl::Codec::set_default_codecs()
97 using google::protobuf::FieldDescriptor;
179 const Descriptor* desc = msg.GetDescriptor();
181 dlog.
is(DEBUG1, ENCODE) && dlog <<
"Began encoding message of type: " << desc->full_name()
186 unsigned dccl_id = (user_id < 0) ?
id(desc) : user_id;
187 size_t head_byte_size = 0;
189 if (!msg.IsInitialized() && !header_only)
191 std::stringstream ss;
193 ss <<
"Message is not properly initialized. All `required` fields must be set. Fields "
195 << get_all_error_fields_in_message(msg);
200 if (!id2desc_.count(dccl_id))
201 throw(
Exception(
"Message id " + std::to_string(dccl_id) +
202 " has not been loaded. Call load() before encoding this type."),
205 std::shared_ptr<FieldCodecBase> codec = manager_.find(desc);
206 std::shared_ptr<internal::FromProtoCppTypeBase> helper = manager_.type_helper().find(desc);
211 id_codec()->field_encode(&head_bits, dccl_id,
nullptr);
214 manager_.codec_data().message_data_);
215 msg_stack.push(msg.GetDescriptor());
216 codec->base_encode(&head_bits, msg, HEAD, strict_);
219 head_byte_size = ceil_bits2bytes(head_bits.size());
220 head_bits.resize(head_byte_size * BITS_IN_BYTE);
224 dlog.
is(DEBUG2, ENCODE) &&
225 dlog <<
"as requested, skipping encoding and encrypting body." << std::endl;
229 codec->base_encode(&body_bits, msg, BODY, strict_);
234 throw(
Exception(
"Failed to find (dccl.msg).codec `" +
235 desc->options().GetExtension(dccl::msg).codec() +
"`"),
241 dlog.
is(DEBUG1, ENCODE) &&
242 dlog <<
"Message " << desc->full_name()
243 <<
" failed to encode because a field was out of bounds and strict == true: "
244 << e.what() << std::endl;
247 catch (std::exception& e)
249 std::stringstream ss;
251 ss <<
"Message " << desc->full_name() <<
" failed to encode. Reason: " << e.what();
253 dlog.
is(DEBUG1, ENCODE) && dlog << ss.str() << std::endl;
259 bool header_only ,
int user_id )
261 const Descriptor* desc = msg.GetDescriptor();
264 encode_internal(msg, header_only, head_bits, body_bits, user_id);
266 size_t head_byte_size = ceil_bits2bytes(head_bits.size());
267 if (max_len < head_byte_size)
269 throw std::length_error(
"max_len must be >= head_byte_size");
273 dlog.
is(DEBUG2, ENCODE) && dlog <<
"Head bytes (bits): " << head_byte_size <<
"("
274 << head_bits.size() <<
")" << std::endl;
275 dlog.
is(DEBUG3, ENCODE) && dlog <<
"Unencrypted Head (bin): " << head_bits << std::endl;
276 dlog.
is(DEBUG3, ENCODE) && dlog <<
"Unencrypted Head (hex): "
277 <<
hex_encode(bytes, bytes + head_byte_size) << std::endl;
279 size_t body_byte_size = 0;
282 body_byte_size = ceil_bits2bytes(body_bits.size());
283 if (max_len < (head_byte_size + body_byte_size))
285 throw std::length_error(
"max_len must be >= (head_byte_size + body_byte_size)");
287 body_bits.
to_byte_string(bytes + head_byte_size, max_len - head_byte_size);
289 dlog.
is(DEBUG3, ENCODE) && dlog <<
"Unencrypted Body (bin): " << body_bits << std::endl;
290 dlog.
is(DEBUG3, ENCODE) &&
291 dlog <<
"Unencrypted Body (hex): "
292 <<
hex_encode(bytes + head_byte_size, bytes + head_byte_size + body_byte_size)
294 dlog.
is(DEBUG2, ENCODE) && dlog <<
"Body bytes (bits): " << body_byte_size <<
"("
295 << body_bits.size() <<
")" << std::endl;
297 unsigned dccl_id = (user_id < 0) ?
id(desc) : user_id;
298 if (!crypto_key_.empty() && !skip_crypto_ids_.count(dccl_id))
300 std::string head_bytes(bytes, bytes + head_byte_size);
301 std::string body_bytes(bytes + head_byte_size, bytes + head_byte_size + body_byte_size);
302 encrypt(&body_bytes, head_bytes);
303 std::memcpy(bytes + head_byte_size, body_bytes.data(), body_bytes.size());
306 dlog.
is(logger::DEBUG3, logger::ENCODE) &&
307 dlog <<
"Encrypted Body (hex): "
308 <<
hex_encode(bytes + head_byte_size, bytes + head_byte_size + body_byte_size)
312 dlog.
is(DEBUG1, ENCODE) && dlog <<
"Successfully encoded message of type: " << desc->full_name()
315 return head_byte_size + body_byte_size;
319 bool header_only ,
int user_id )
321 const Descriptor* desc = msg.GetDescriptor();
324 encode_internal(msg, header_only, head_bits, body_bits, user_id);
328 dlog.
is(DEBUG2, ENCODE) && dlog <<
"Head bytes (bits): " << head_bytes.size() <<
"("
329 << head_bits.size() <<
")" << std::endl;
330 dlog.
is(DEBUG3, ENCODE) && dlog <<
"Unencrypted Head (bin): " << head_bits << std::endl;
331 dlog.
is(DEBUG3, ENCODE) && dlog <<
"Unencrypted Head (hex): " <<
hex_encode(head_bytes)
334 std::string body_bytes;
339 dlog.
is(DEBUG3, ENCODE) && dlog <<
"Unencrypted Body (bin): " << body_bits << std::endl;
340 dlog.
is(DEBUG3, ENCODE) && dlog <<
"Unencrypted Body (hex): " <<
hex_encode(body_bytes)
342 dlog.
is(DEBUG2, ENCODE) && dlog <<
"Body bytes (bits): " << body_bytes.size() <<
"("
343 << body_bits.size() <<
")" << std::endl;
345 unsigned dccl_id = (user_id < 0) ?
id(desc) : user_id;
346 if (!crypto_key_.empty() && !skip_crypto_ids_.count(dccl_id))
347 encrypt(&body_bytes, head_bytes);
349 dlog.
is(logger::DEBUG3, logger::ENCODE) &&
350 dlog <<
"Encrypted Body (hex): " <<
hex_encode(body_bytes) << std::endl;
353 dlog.
is(DEBUG1, ENCODE) && dlog <<
"Successfully encoded message of type: " << desc->full_name()
355 *bytes += head_bytes + body_bytes;
358 unsigned dccl::Codec::id(
const std::string& bytes)
const {
return id(bytes.begin(), bytes.end()); }
363 unsigned last_size = size(*msg);
364 bytes->erase(0, last_size);
370 decode(bytes.begin(), bytes.end(), msg, header_only);
379 if (user_id < 0 && !desc->options().GetExtension(dccl::msg).has_id())
381 Exception(
"Missing message option `(dccl.msg).id`. Specify a unique id (e.g. 3) in "
382 "the body of your .proto message using \"option (dccl.msg).id = 3\"",
384 if (!desc->options().GetExtension(dccl::msg).has_max_bytes())
385 throw(
Exception(
"Missing message option `(dccl.msg).max_bytes`. Specify a maximum "
386 "(encoded) message size in bytes (e.g. 32) in the body of your .proto "
387 "message using \"option (dccl.msg).max_bytes = 32\"",
390 if (!desc->options().GetExtension(dccl::msg).has_codec_version())
392 "No (dccl.msg).codec_version set for DCCL Message '" + desc->full_name() +
393 "'. For new messages, set 'option (dccl.msg).codec_version = 4' in the "
394 "message definition for " +
395 desc->full_name() +
" to use the default DCCL4 codecs.",
398 std::shared_ptr<FieldCodecBase> codec = manager_.find(desc);
400 unsigned dccl_id = (user_id < 0) ?
id(desc) : user_id;
401 unsigned head_size_bits, body_size_bits;
402 codec->base_max_size(&head_size_bits, desc, HEAD);
403 codec->base_max_size(&body_size_bits, desc, BODY);
405 unsigned id_bits = 0;
406 id_codec()->field_size(&id_bits, dccl_id,
nullptr);
407 head_size_bits += id_bits;
409 const unsigned byte_size =
410 ceil_bits2bytes(head_size_bits) + ceil_bits2bytes(body_size_bits);
412 auto actual_max = byte_size;
413 auto allowed_max = desc->options().GetExtension(dccl::msg).max_bytes();
414 if (actual_max > allowed_max)
415 throw(
Exception(
"Actual maximum size of message (" + std::to_string(actual_max) +
416 "B) exceeds allowed maximum (" + std::to_string(allowed_max) +
418 "bounds, remove fields, improve codecs, or increase the allowed "
419 "value in (dccl.msg).max_bytes",
422 codec->base_validate(desc, HEAD);
423 codec->base_validate(desc, BODY);
425 if (id2desc_.count(dccl_id) && desc != id2desc_.find(dccl_id)->second)
427 std::stringstream ss;
428 ss <<
"`dccl id` " << dccl_id <<
" is already in use by Message "
429 << id2desc_.find(dccl_id)->second->full_name() <<
": "
430 << id2desc_.find(dccl_id)->second;
436 id2desc_.insert(std::make_pair(dccl_id, desc));
439 dlog.
is(DEBUG1) && dlog <<
"Successfully validated message of type: " << desc->full_name()
452 dlog.
is(DEBUG1) && dlog <<
"Message " << desc->full_name() <<
": " << desc
453 <<
" failed validation. Reason: " << e.what() <<
"\n"
454 <<
"If possible, information about the Message are printed above. "
463 unsigned int erased = 0;
464 for (
auto it = id2desc_.begin(); it != id2desc_.end();)
466 if (it->second == desc)
469 id2desc_.erase(it++);
478 dlog.
is(DEBUG1) && dlog <<
"Message " << desc->full_name()
479 <<
": is not loaded. Ignoring unload request." << std::endl;
485 if (id2desc_.count(dccl_id))
487 id2desc_.erase(dccl_id);
491 dlog.
is(DEBUG1) && dlog <<
"Message with id " << dccl_id
492 <<
": is not loaded. Ignoring unload request." << std::endl;
499 const Descriptor* desc = msg.GetDescriptor();
501 std::shared_ptr<FieldCodecBase> codec = manager_.find(desc);
503 unsigned dccl_id = (user_id < 0) ?
id(desc) : user_id;
504 unsigned head_size_bits;
505 codec->base_size(&head_size_bits, msg, HEAD);
507 unsigned id_bits = 0;
508 id_codec()->field_size(&id_bits, dccl_id,
nullptr);
509 head_size_bits += id_bits;
511 unsigned body_size_bits;
512 codec->base_size(&body_size_bits, msg, BODY);
514 const unsigned head_size_bytes = ceil_bits2bytes(head_size_bits);
515 const unsigned body_size_bytes = ceil_bits2bytes(body_size_bits);
516 return head_size_bytes + body_size_bytes;
521 std::shared_ptr<FieldCodecBase> codec = manager_.find(desc);
523 unsigned head_size_bits;
524 codec->base_max_size(&head_size_bits, desc, HEAD);
526 unsigned id_bits = 0;
527 id_codec()->field_max_size(&id_bits,
nullptr);
528 head_size_bits += id_bits;
530 unsigned body_size_bits;
531 codec->base_max_size(&body_size_bits, desc, BODY);
533 const unsigned head_size_bytes = ceil_bits2bytes(head_size_bits);
534 const unsigned body_size_bytes = ceil_bits2bytes(body_size_bits);
535 return head_size_bytes + body_size_bytes;
540 std::shared_ptr<FieldCodecBase> codec = manager_.find(desc);
542 unsigned head_size_bits;
543 codec->base_min_size(&head_size_bits, desc, HEAD);
545 unsigned id_bits = 0;
546 id_codec()->field_min_size(&id_bits,
nullptr);
547 head_size_bits += id_bits;
549 unsigned body_size_bits;
550 codec->base_min_size(&body_size_bits, desc, BODY);
552 const unsigned head_size_bytes = ceil_bits2bytes(head_size_bits);
553 const unsigned body_size_bytes = ceil_bits2bytes(body_size_bits);
554 return head_size_bytes + body_size_bytes;
560 bool is_dlog =
false;
561 if (!param_os || param_os == &dlog)
563 std::stringstream ss;
564 std::ostream* os = (is_dlog) ? &ss : param_os;
566 if (!is_dlog || dlog.
check(INFO))
570 std::shared_ptr<FieldCodecBase> codec = manager_.find(desc);
572 unsigned config_head_bit_size, body_bit_size;
573 codec->base_max_size(&config_head_bit_size, desc, HEAD);
574 codec->base_max_size(&body_bit_size, desc, BODY);
576 unsigned dccl_id = (user_id < 0) ?
id(desc) : user_id;
577 unsigned id_bit_size = 0;
578 id_codec()->field_size(&id_bit_size, dccl_id,
nullptr);
580 const unsigned bit_size = id_bit_size + config_head_bit_size + body_bit_size;
582 const unsigned byte_size = ceil_bits2bytes(config_head_bit_size + id_bit_size) +
583 ceil_bits2bytes(body_bit_size);
585 const unsigned allowed_byte_size = desc->options().GetExtension(dccl::msg).max_bytes();
586 const unsigned allowed_bit_size = allowed_byte_size * BITS_IN_BYTE;
588 std::string message_name = std::to_string(dccl_id) +
": " + desc->full_name();
589 std::string guard = build_guard_for_console_output(message_name,
'=');
590 std::string bits_dccl_head_str =
"dccl.id head";
591 std::string bits_user_head_str =
"user head";
592 std::string bits_body_str =
"body";
593 std::string bits_padding_str =
"padding to full byte";
595 const int bits_width = 40;
596 const int spaces = 8;
597 std::string indent = std::string(spaces,
' ');
599 *os << guard <<
" " << message_name <<
" " << guard <<
"\n"
600 <<
"Actual maximum size of message: " << byte_size <<
" bytes / "
601 << byte_size * BITS_IN_BYTE <<
" bits\n"
602 << indent << bits_dccl_head_str << std::setfill(
'.')
603 << std::setw(bits_width - bits_dccl_head_str.size()) << id_bit_size <<
"\n"
604 << indent << bits_user_head_str << std::setfill(
'.')
605 << std::setw(bits_width - bits_user_head_str.size()) << config_head_bit_size <<
"\n"
606 << indent << bits_body_str << std::setfill(
'.')
607 << std::setw(bits_width - bits_body_str.size()) << body_bit_size <<
"\n"
608 << indent << bits_padding_str << std::setfill(
'.')
609 << std::setw(bits_width - bits_padding_str.size())
610 << byte_size * BITS_IN_BYTE - bit_size <<
"\n"
611 <<
"Allowed maximum size of message: " << allowed_byte_size <<
" bytes / "
612 << allowed_bit_size <<
" bits\n";
614 std::string header_str =
"Header";
615 std::string header_guard = build_guard_for_console_output(header_str,
'-');
617 *os << header_guard <<
" " << header_str <<
" " << header_guard <<
"\n";
618 *os << bits_dccl_head_str << std::setfill(
'.')
619 << std::setw(bits_width - bits_dccl_head_str.size() + spaces) << id_bit_size <<
" {"
620 << id_codec()->name() <<
"}\n";
621 codec->base_info(os, desc, HEAD);
624 std::string body_str =
"Body";
625 std::string body_guard = build_guard_for_console_output(body_str,
'-');
627 *os << body_guard <<
" " << body_str <<
" " << body_guard <<
"\n";
628 codec->base_info(os, desc, BODY);
635 dlog.
is(INFO) && dlog << ss.str() << std::endl;
640 dlog <<
"Message " << desc->full_name()
641 <<
" cannot provide information due to invalid configuration. Reason: "
642 << e.what() << std::endl;
647 void dccl::Codec::encrypt(std::string* s,
const std::string& nonce )
649 #if DCCL_HAS_CRYPTOPP
650 using namespace CryptoPP;
654 StringSource unused(nonce,
true,
new HashFilter(hash,
new StringSink(iv)));
656 CTR_Mode<AES>::Encryption encryptor;
657 encryptor.SetKeyWithIV((
byte*)crypto_key_.c_str(), crypto_key_.size(), (
byte*)iv.c_str());
660 StreamTransformationFilter in(encryptor,
new StringSink(cipher));
661 in.Put((
byte*)s->c_str(), s->size());
667 void dccl::Codec::decrypt(std::string* s,
const std::string& nonce)
669 #if DCCL_HAS_CRYPTOPP
670 using namespace CryptoPP;
674 StringSource unused(nonce,
true,
new HashFilter(hash,
new StringSink(iv)));
676 CTR_Mode<AES>::Decryption decryptor;
677 decryptor.SetKeyWithIV((
byte*)crypto_key_.c_str(), crypto_key_.size(), (
byte*)iv.c_str());
679 std::string recovered;
681 StreamTransformationFilter out(decryptor,
new StringSink(recovered));
682 out.Put((
byte*)s->c_str(), s->size());
690 void* handle = dlopen(library_path.c_str(), RTLD_LAZY);
692 dl_handles_.push_back(handle);
693 load_library(handle);
699 throw(
Exception(
"Null shared library handle passed to load_library"));
703 dccl_load_ptr = (void (*)(
dccl::Codec*))dlsym(dl_handle,
"dccl3_load");
705 (*dccl_load_ptr)(
this);
711 throw(
Exception(
"Null shared library handle passed to unload_library"));
715 dccl_unload_ptr = (void (*)(
dccl::Codec*))dlsym(dl_handle,
"dccl3_unload");
717 (*dccl_unload_ptr)(
this);
721 const std::string& passphrase,
722 const std::set<unsigned>& do_not_encrypt_ids_ )
724 if (!crypto_key_.empty())
726 skip_crypto_ids_.clear();
728 #if DCCL_HAS_CRYPTOPP
729 using namespace CryptoPP;
732 StringSource unused(passphrase,
true,
new HashFilter(hash,
new StringSink(crypto_key_)));
734 dlog.
is(DEBUG1) && dlog <<
"Cryptography enabled with given passphrase" << std::endl;
736 dlog.
is(DEBUG1) && dlog <<
"Cryptography disabled because DCCL was compiled without support of "
737 "Crypto++. Install Crypto++ and recompile to enable cryptography."
741 skip_crypto_ids_ = do_not_encrypt_ids_;
746 bool is_dlog =
false;
747 if (!param_os || param_os == &dlog)
749 std::stringstream ss;
750 std::ostream* os = (is_dlog) ? &ss : param_os;
751 if (!is_dlog || dlog.
check(INFO))
753 std::string codec_str =
"Dynamic Compact Control Language (DCCL) Codec";
754 std::string codec_guard = build_guard_for_console_output(codec_str,
'|');
756 *os << codec_guard <<
" " << codec_str <<
" " << codec_guard <<
"\n";
757 *os << id2desc_.size() <<
" messages loaded.\n";
758 *os <<
"Field sizes are in bits unless otherwise noted."
761 for (
auto it : id2desc_) info(it.second, os, it.first);
765 dlog.
is(INFO) && dlog << ss.str() << std::endl;
774 id_codec_ = id_codec_name;
779 std::string dccl::Codec::build_guard_for_console_output(std::string& base,
char guard_char)
const
782 return (base.size() < console_width_)
783 ? std::string((console_width_ - base.size()) / 2, guard_char)
795 std::stringstream output_stream;
797 const google::protobuf::Descriptor* descriptor = message.GetDescriptor();
798 const google::protobuf::Reflection* reflection = message.GetReflection();
799 const uint8_t depth_spacing = 4;
802 const int32_t field_count = descriptor->field_count();
804 for (int32_t index = 0; index < field_count; ++index)
806 if (descriptor->field(index)->is_required())
808 if (!reflection->HasField(message, descriptor->field(index)))
810 output_stream << std::string(depth * depth_spacing,
' ')
811 << descriptor->field(index)->number() <<
": "
812 << descriptor->field(index)->name() <<
"\n";
818 std::vector<const google::protobuf::FieldDescriptor*> fields;
819 reflection->ListFields(message, &fields);
821 for (
const google::protobuf::FieldDescriptor* field : fields)
823 if (field->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE)
825 output_stream << std::string(depth * depth_spacing,
' ') << field->number() <<
": "
826 << field->name() <<
"\n";
828 if (field->is_repeated())
830 int32_t size = reflection->FieldSize(message, field);
832 for (int32_t index = 0; index < size; ++index)
835 reflection->GetRepeatedMessage(message, field, index);
837 output_stream << std::string((depth + 1) * depth_spacing,
' ') <<
"[" << index
841 output_stream << get_all_error_fields_in_message(sub_message, depth + 2);
847 reflection->GetMessage(message, field);
848 output_stream << get_all_error_fields_in_message(sub_message, depth + 1);
853 return output_stream.str();