DCCL v4
codec.cpp
1 // Copyright 2009-2023:
2 // GobySoft, LLC (2013-)
3 // Massachusetts Institute of Technology (2007-2014)
4 // Community contributors (see AUTHORS file)
5 // File authors:
6 // Toby Schneider <toby@gobysoft.org>
7 // philboske <philboske@gmail.com>
8 // Denis <dbiryukov@slb.com>
9 // Nathan Knotts <nknotts@gmail.com>
10 // Chris Murphy <cmurphy@aphysci.com>
11 // Davide Fenucci <davfen@noc.ac.uk>
12 // Kyle Guilbert <kguilbert@aphysci.com>
13 //
14 //
15 // This file is part of the Dynamic Compact Control Language Library
16 // ("DCCL").
17 //
18 // DCCL is free software: you can redistribute it and/or modify
19 // it under the terms of the GNU Lesser General Public License as published by
20 // the Free Software Foundation, either version 2.1 of the License, or
21 // (at your option) any later version.
22 //
23 // DCCL is distributed in the hope that it will be useful,
24 // but WITHOUT ANY WARRANTY; without even the implied warranty of
25 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26 // GNU Lesser General Public License for more details.
27 //
28 // You should have received a copy of the GNU Lesser General Public License
29 // along with DCCL. If not, see <http://www.gnu.org/licenses/>.
30 #include <algorithm>
31 #include <utility>
32 
33 #include <dlfcn.h> // for shared library loading
34 
35 #include "codec.h"
36 
37 #if DCCL_HAS_CRYPTOPP
38 #if CRYPTOPP_PATH_USES_PLUS_SIGN
39 #include <crypto++/aes.h>
40 #include <crypto++/filters.h>
41 #include <crypto++/modes.h>
42 #include <crypto++/sha.h>
43 #else
44 #include <cryptopp/aes.h>
45 #include <cryptopp/filters.h>
46 #include <cryptopp/modes.h>
47 #include <cryptopp/sha.h>
48 #endif // CRYPTOPP_PATH_USES_PLUS_SIGN
49 #endif // HAS_CRYPTOPP
50 
51 #include "codecs2/field_codec_default.h"
52 #include "codecs3/field_codec_default.h"
53 #include "codecs3/field_codec_presence.h"
54 #include "codecs3/field_codec_var_bytes.h"
55 #include "codecs4/field_codec_default.h"
56 #include "codecs4/field_codec_default_message.h"
57 #include "field_codec_id.h"
58 
59 #include "option_extensions.pb.h"
60 
61 using dccl::hex_encode;
62 
63 using namespace dccl;
64 using namespace dccl::logger;
65 
66 using google::protobuf::Descriptor;
67 using google::protobuf::FieldDescriptor;
68 using google::protobuf::Reflection;
69 
70 //
71 // Codec
72 //
73 
74 dccl::Codec::Codec(std::string dccl_id_codec_name, const std::string& library_path)
75  : id_codec_(std::move(dccl_id_codec_name))
76 {
77  set_default_codecs();
78  manager_.add<DefaultIdentifierCodec>(default_id_codec_name());
79 
80  if (!library_path.empty())
81  load_library(library_path);
82  // make sure the id codec exists
83  id_codec();
84 }
85 
86 dccl::Codec::~Codec()
87 {
88  for (auto& dl_handle : dl_handles_)
89  {
90  unload_library(dl_handle);
91  dlclose(dl_handle);
92  }
93 }
94 
95 void dccl::Codec::set_default_codecs()
96 {
97  using google::protobuf::FieldDescriptor;
98 
99  // version 2
100  manager_.add<v2::DefaultNumericFieldCodec<double>>(default_codec_name());
101  manager_.add<v2::DefaultNumericFieldCodec<float>>(default_codec_name());
102  manager_.add<v2::DefaultBoolCodec>(default_codec_name());
103  manager_.add<v2::DefaultNumericFieldCodec<int32>>(default_codec_name());
104  manager_.add<v2::DefaultNumericFieldCodec<int64>>(default_codec_name());
105  manager_.add<v2::DefaultNumericFieldCodec<uint32>>(default_codec_name());
106  manager_.add<v2::DefaultNumericFieldCodec<uint64>>(default_codec_name());
107  manager_.add<v2::DefaultStringCodec, FieldDescriptor::TYPE_STRING>(default_codec_name());
108  manager_.add<v2::DefaultBytesCodec, FieldDescriptor::TYPE_BYTES>(default_codec_name());
109  manager_.add<v2::DefaultEnumCodec>(default_codec_name());
110  manager_.add<v2::DefaultMessageCodec, FieldDescriptor::TYPE_MESSAGE>(default_codec_name());
111 
112  manager_.add<v2::TimeCodec<uint64>>("dccl.time2");
113  manager_.add<v2::TimeCodec<int64>>("dccl.time2");
114  manager_.add<v2::TimeCodec<double>>("dccl.time2");
115 
116  manager_.add<v2::StaticCodec<std::string>>("dccl.static2");
117  manager_.add<v2::StaticCodec<double>>("dccl.static2");
118  manager_.add<v2::StaticCodec<float>>("dccl.static2");
119  manager_.add<v2::StaticCodec<int32>>("dccl.static2");
120  manager_.add<v2::StaticCodec<int64>>("dccl.static2");
121  manager_.add<v2::StaticCodec<uint32>>("dccl.static2");
122  manager_.add<v2::StaticCodec<uint64>>("dccl.static2");
123 
124  // version 3
125  manager_.add<v3::DefaultNumericFieldCodec<double>>(default_codec_name(3));
126  manager_.add<v3::DefaultNumericFieldCodec<float>>(default_codec_name(3));
127  manager_.add<v3::DefaultBoolCodec>(default_codec_name(3));
128  manager_.add<v3::DefaultNumericFieldCodec<int32>>(default_codec_name(3));
129  manager_.add<v3::DefaultNumericFieldCodec<int64>>(default_codec_name(3));
130  manager_.add<v3::DefaultNumericFieldCodec<uint32>>(default_codec_name(3));
131  manager_.add<v3::DefaultNumericFieldCodec<uint64>>(default_codec_name(3));
132  manager_.add<v3::DefaultStringCodec, FieldDescriptor::TYPE_STRING>(default_codec_name(3));
133  manager_.add<v3::DefaultBytesCodec, FieldDescriptor::TYPE_BYTES>(default_codec_name(3));
134  manager_.add<v3::DefaultEnumCodec>(default_codec_name(3));
135  manager_.add<v3::DefaultMessageCodec, FieldDescriptor::TYPE_MESSAGE>(default_codec_name(3));
136 
137  // version 4
138  manager_.add<v4::DefaultNumericFieldCodec<double>>(default_codec_name(4));
139  manager_.add<v4::DefaultNumericFieldCodec<float>>(default_codec_name(4));
140  manager_.add<v4::DefaultBoolCodec>(default_codec_name(4));
141  manager_.add<v4::DefaultNumericFieldCodec<int32>>(default_codec_name(4));
142  manager_.add<v4::DefaultNumericFieldCodec<int64>>(default_codec_name(4));
143  manager_.add<v4::DefaultNumericFieldCodec<uint32>>(default_codec_name(4));
144  manager_.add<v4::DefaultNumericFieldCodec<uint64>>(default_codec_name(4));
145  manager_.add<v4::DefaultStringCodec, FieldDescriptor::TYPE_STRING>(default_codec_name(4));
146  manager_.add<v4::DefaultBytesCodec, FieldDescriptor::TYPE_BYTES>(default_codec_name(4));
147  manager_.add<v4::DefaultEnumCodec>(default_codec_name(4));
148  manager_.add<v4::DefaultMessageCodec, FieldDescriptor::TYPE_MESSAGE>(default_codec_name(4));
149 
150  // presence bit codec, which encode empty optional fields with a single bit
151  manager_.add<v3::PresenceBitCodec<v3::DefaultNumericFieldCodec<double>>>("dccl.presence");
152  manager_.add<v3::PresenceBitCodec<v3::DefaultNumericFieldCodec<float>>>("dccl.presence");
153  manager_.add<v3::PresenceBitCodec<v3::DefaultNumericFieldCodec<int32>>>("dccl.presence");
154  manager_.add<v3::PresenceBitCodec<v3::DefaultNumericFieldCodec<int64>>>("dccl.presence");
155  manager_.add<v3::PresenceBitCodec<v3::DefaultNumericFieldCodec<uint32>>>("dccl.presence");
156  manager_.add<v3::PresenceBitCodec<v3::DefaultNumericFieldCodec<uint64>>>("dccl.presence");
157  manager_.add<v3::PresenceBitCodec<v3::DefaultEnumCodec>>("dccl.presence");
158 
159  // alternative bytes codec that more efficiently encodes variable length bytes fields
160  manager_.add<v3::VarBytesCodec, FieldDescriptor::TYPE_BYTES>("dccl.var_bytes");
161 
162  // for backwards compatibility
163  manager_.add<v2::TimeCodec<uint64>>("_time");
164  manager_.add<v2::TimeCodec<int64>>("_time");
165  manager_.add<v2::TimeCodec<double>>("_time");
166 
167  manager_.add<v2::StaticCodec<std::string>>("_static");
168  manager_.add<v2::StaticCodec<double>>("_static");
169  manager_.add<v2::StaticCodec<float>>("_static");
170  manager_.add<v2::StaticCodec<int32>>("_static");
171  manager_.add<v2::StaticCodec<int64>>("_static");
172  manager_.add<v2::StaticCodec<uint32>>("_static");
173  manager_.add<v2::StaticCodec<uint64>>("_static");
174 }
175 
176 void dccl::Codec::encode_internal(const google::protobuf::Message& msg, bool header_only,
177  Bitset& head_bits, Bitset& body_bits, int user_id)
178 {
179  const Descriptor* desc = msg.GetDescriptor();
180 
181  dlog.is(DEBUG1, ENCODE) && dlog << "Began encoding message of type: " << desc->full_name()
182  << std::endl;
183 
184  try
185  {
186  unsigned dccl_id = (user_id < 0) ? id(desc) : user_id;
187  size_t head_byte_size = 0;
188 
189  if (!msg.IsInitialized() && !header_only)
190  {
191  std::stringstream ss;
192 
193  ss << "Message is not properly initialized. All `required` fields must be set. Fields "
194  "with errors: \n"
195  << get_all_error_fields_in_message(msg);
196 
197  throw(Exception(ss.str(), desc));
198  }
199 
200  if (!id2desc_.count(dccl_id))
201  throw(Exception("Message id " + std::to_string(dccl_id) +
202  " has not been loaded. Call load() before encoding this type."),
203  desc);
204 
205  std::shared_ptr<FieldCodecBase> codec = manager_.find(desc);
206  std::shared_ptr<internal::FromProtoCppTypeBase> helper = manager_.type_helper().find(desc);
207 
208  if (codec)
209  {
210  //fixed header
211  id_codec()->field_encode(&head_bits, dccl_id, nullptr);
212 
213  internal::MessageStack msg_stack(manager_.codec_data().root_message_,
214  manager_.codec_data().message_data_);
215  msg_stack.push(msg.GetDescriptor());
216  codec->base_encode(&head_bits, msg, HEAD, strict_);
217 
218  // given header of not even byte size (e.g. 01011), make even byte size (e.g. 00001011)
219  head_byte_size = ceil_bits2bytes(head_bits.size());
220  head_bits.resize(head_byte_size * BITS_IN_BYTE);
221 
222  if (header_only)
223  {
224  dlog.is(DEBUG2, ENCODE) &&
225  dlog << "as requested, skipping encoding and encrypting body." << std::endl;
226  }
227  else
228  {
229  codec->base_encode(&body_bits, msg, BODY, strict_);
230  }
231  }
232  else
233  {
234  throw(Exception("Failed to find (dccl.msg).codec `" +
235  desc->options().GetExtension(dccl::msg).codec() + "`"),
236  desc);
237  }
238  }
239  catch (dccl::OutOfRangeException& e)
240  {
241  dlog.is(DEBUG1, ENCODE) &&
242  dlog << "Message " << desc->full_name()
243  << " failed to encode because a field was out of bounds and strict == true: "
244  << e.what() << std::endl;
245  throw;
246  }
247  catch (std::exception& e)
248  {
249  std::stringstream ss;
250 
251  ss << "Message " << desc->full_name() << " failed to encode. Reason: " << e.what();
252 
253  dlog.is(DEBUG1, ENCODE) && dlog << ss.str() << std::endl;
254  throw(Exception(ss.str(), desc));
255  }
256 }
257 
258 size_t dccl::Codec::encode(char* bytes, size_t max_len, const google::protobuf::Message& msg,
259  bool header_only /* = false */, int user_id /* = -1 */)
260 {
261  const Descriptor* desc = msg.GetDescriptor();
262  Bitset head_bits;
263  Bitset body_bits;
264  encode_internal(msg, header_only, head_bits, body_bits, user_id);
265 
266  size_t head_byte_size = ceil_bits2bytes(head_bits.size());
267  if (max_len < head_byte_size)
268  {
269  throw std::length_error("max_len must be >= head_byte_size");
270  }
271  head_bits.to_byte_string(bytes, head_byte_size);
272 
273  dlog.is(DEBUG2, ENCODE) && dlog << "Head bytes (bits): " << head_byte_size << "("
274  << head_bits.size() << ")" << std::endl;
275  dlog.is(DEBUG3, ENCODE) && dlog << "Unencrypted Head (bin): " << head_bits << std::endl;
276  dlog.is(DEBUG3, ENCODE) && dlog << "Unencrypted Head (hex): "
277  << hex_encode(bytes, bytes + head_byte_size) << std::endl;
278 
279  size_t body_byte_size = 0;
280  if (!header_only)
281  {
282  body_byte_size = ceil_bits2bytes(body_bits.size());
283  if (max_len < (head_byte_size + body_byte_size))
284  {
285  throw std::length_error("max_len must be >= (head_byte_size + body_byte_size)");
286  }
287  body_bits.to_byte_string(bytes + head_byte_size, max_len - head_byte_size);
288 
289  dlog.is(DEBUG3, ENCODE) && dlog << "Unencrypted Body (bin): " << body_bits << std::endl;
290  dlog.is(DEBUG3, ENCODE) &&
291  dlog << "Unencrypted Body (hex): "
292  << hex_encode(bytes + head_byte_size, bytes + head_byte_size + body_byte_size)
293  << std::endl;
294  dlog.is(DEBUG2, ENCODE) && dlog << "Body bytes (bits): " << body_byte_size << "("
295  << body_bits.size() << ")" << std::endl;
296 
297  unsigned dccl_id = (user_id < 0) ? id(desc) : user_id;
298  if (!crypto_key_.empty() && !skip_crypto_ids_.count(dccl_id))
299  {
300  std::string head_bytes(bytes, bytes + head_byte_size);
301  std::string body_bytes(bytes + head_byte_size, bytes + head_byte_size + body_byte_size);
302  encrypt(&body_bytes, head_bytes);
303  std::memcpy(bytes + head_byte_size, body_bytes.data(), body_bytes.size());
304  }
305 
306  dlog.is(logger::DEBUG3, logger::ENCODE) &&
307  dlog << "Encrypted Body (hex): "
308  << hex_encode(bytes + head_byte_size, bytes + head_byte_size + body_byte_size)
309  << std::endl;
310  }
311 
312  dlog.is(DEBUG1, ENCODE) && dlog << "Successfully encoded message of type: " << desc->full_name()
313  << std::endl;
314 
315  return head_byte_size + body_byte_size;
316 }
317 
318 void dccl::Codec::encode(std::string* bytes, const google::protobuf::Message& msg,
319  bool header_only /* = false */, int user_id /* = -1 */)
320 {
321  const Descriptor* desc = msg.GetDescriptor();
322  Bitset head_bits;
323  Bitset body_bits;
324  encode_internal(msg, header_only, head_bits, body_bits, user_id);
325 
326  std::string head_bytes = head_bits.to_byte_string();
327 
328  dlog.is(DEBUG2, ENCODE) && dlog << "Head bytes (bits): " << head_bytes.size() << "("
329  << head_bits.size() << ")" << std::endl;
330  dlog.is(DEBUG3, ENCODE) && dlog << "Unencrypted Head (bin): " << head_bits << std::endl;
331  dlog.is(DEBUG3, ENCODE) && dlog << "Unencrypted Head (hex): " << hex_encode(head_bytes)
332  << std::endl;
333 
334  std::string body_bytes;
335  if (!header_only)
336  {
337  body_bytes = body_bits.to_byte_string();
338 
339  dlog.is(DEBUG3, ENCODE) && dlog << "Unencrypted Body (bin): " << body_bits << std::endl;
340  dlog.is(DEBUG3, ENCODE) && dlog << "Unencrypted Body (hex): " << hex_encode(body_bytes)
341  << std::endl;
342  dlog.is(DEBUG2, ENCODE) && dlog << "Body bytes (bits): " << body_bytes.size() << "("
343  << body_bits.size() << ")" << std::endl;
344 
345  unsigned dccl_id = (user_id < 0) ? id(desc) : user_id;
346  if (!crypto_key_.empty() && !skip_crypto_ids_.count(dccl_id))
347  encrypt(&body_bytes, head_bytes);
348 
349  dlog.is(logger::DEBUG3, logger::ENCODE) &&
350  dlog << "Encrypted Body (hex): " << hex_encode(body_bytes) << std::endl;
351  }
352 
353  dlog.is(DEBUG1, ENCODE) && dlog << "Successfully encoded message of type: " << desc->full_name()
354  << std::endl;
355  *bytes += head_bytes + body_bytes;
356 }
357 
358 unsigned dccl::Codec::id(const std::string& bytes) const { return id(bytes.begin(), bytes.end()); }
359 
360 void dccl::Codec::decode(std::string* bytes, google::protobuf::Message* msg)
361 {
362  decode(*bytes, msg);
363  unsigned last_size = size(*msg);
364  bytes->erase(0, last_size);
365 }
366 
367 void dccl::Codec::decode(const std::string& bytes, google::protobuf::Message* msg,
368  bool header_only /* = false */)
369 {
370  decode(bytes.begin(), bytes.end(), msg, header_only);
371 }
372 
373 // makes sure we can actual encode / decode a message of this descriptor given the loaded FieldCodecs
374 // checks all bounds on the message
375 void dccl::Codec::load(const google::protobuf::Descriptor* desc, int user_id /* = -1 */)
376 {
377  try
378  {
379  if (user_id < 0 && !desc->options().GetExtension(dccl::msg).has_id())
380  throw(
381  Exception("Missing message option `(dccl.msg).id`. Specify a unique id (e.g. 3) in "
382  "the body of your .proto message using \"option (dccl.msg).id = 3\"",
383  desc));
384  if (!desc->options().GetExtension(dccl::msg).has_max_bytes())
385  throw(Exception("Missing message option `(dccl.msg).max_bytes`. Specify a maximum "
386  "(encoded) message size in bytes (e.g. 32) in the body of your .proto "
387  "message using \"option (dccl.msg).max_bytes = 32\"",
388  desc));
389 
390  if (!desc->options().GetExtension(dccl::msg).has_codec_version())
391  throw(Exception(
392  "No (dccl.msg).codec_version set for DCCL Message '" + desc->full_name() +
393  "'. For new messages, set 'option (dccl.msg).codec_version = 4' in the "
394  "message definition for " +
395  desc->full_name() + " to use the default DCCL4 codecs.",
396  desc));
397 
398  std::shared_ptr<FieldCodecBase> codec = manager_.find(desc);
399 
400  unsigned dccl_id = (user_id < 0) ? id(desc) : user_id;
401  unsigned head_size_bits, body_size_bits;
402  codec->base_max_size(&head_size_bits, desc, HEAD);
403  codec->base_max_size(&body_size_bits, desc, BODY);
404 
405  unsigned id_bits = 0;
406  id_codec()->field_size(&id_bits, dccl_id, nullptr);
407  head_size_bits += id_bits;
408 
409  const unsigned byte_size =
410  ceil_bits2bytes(head_size_bits) + ceil_bits2bytes(body_size_bits);
411 
412  auto actual_max = byte_size;
413  auto allowed_max = desc->options().GetExtension(dccl::msg).max_bytes();
414  if (actual_max > allowed_max)
415  throw(Exception("Actual maximum size of message (" + std::to_string(actual_max) +
416  "B) exceeds allowed maximum (" + std::to_string(allowed_max) +
417  "B). Tighten "
418  "bounds, remove fields, improve codecs, or increase the allowed "
419  "value in (dccl.msg).max_bytes",
420  desc));
421 
422  codec->base_validate(desc, HEAD);
423  codec->base_validate(desc, BODY);
424 
425  if (id2desc_.count(dccl_id) && desc != id2desc_.find(dccl_id)->second)
426  {
427  std::stringstream ss;
428  ss << "`dccl id` " << dccl_id << " is already in use by Message "
429  << id2desc_.find(dccl_id)->second->full_name() << ": "
430  << id2desc_.find(dccl_id)->second;
431 
432  throw(Exception(ss.str(), desc));
433  }
434  else
435  {
436  id2desc_.insert(std::make_pair(dccl_id, desc));
437  }
438 
439  dlog.is(DEBUG1) && dlog << "Successfully validated message of type: " << desc->full_name()
440  << std::endl;
441  }
442  catch (Exception& e)
443  {
444  try
445  {
446  info(desc, &dlog);
447  }
448  catch (Exception& e)
449  {
450  }
451 
452  dlog.is(DEBUG1) && dlog << "Message " << desc->full_name() << ": " << desc
453  << " failed validation. Reason: " << e.what() << "\n"
454  << "If possible, information about the Message are printed above. "
455  << std::endl;
456 
457  throw;
458  }
459 }
460 
461 void dccl::Codec::unload(const google::protobuf::Descriptor* desc)
462 {
463  unsigned int erased = 0;
464  for (auto it = id2desc_.begin(); it != id2desc_.end();)
465  {
466  if (it->second == desc)
467  {
468  erased++;
469  id2desc_.erase(it++);
470  }
471  else
472  {
473  it++;
474  }
475  }
476  if (erased == 0)
477  {
478  dlog.is(DEBUG1) && dlog << "Message " << desc->full_name()
479  << ": is not loaded. Ignoring unload request." << std::endl;
480  }
481 }
482 
483 void dccl::Codec::unload(size_t dccl_id)
484 {
485  if (id2desc_.count(dccl_id))
486  {
487  id2desc_.erase(dccl_id);
488  }
489  else
490  {
491  dlog.is(DEBUG1) && dlog << "Message with id " << dccl_id
492  << ": is not loaded. Ignoring unload request." << std::endl;
493  return;
494  }
495 }
496 
497 unsigned dccl::Codec::size(const google::protobuf::Message& msg, int user_id /* = -1 */)
498 {
499  const Descriptor* desc = msg.GetDescriptor();
500 
501  std::shared_ptr<FieldCodecBase> codec = manager_.find(desc);
502 
503  unsigned dccl_id = (user_id < 0) ? id(desc) : user_id;
504  unsigned head_size_bits;
505  codec->base_size(&head_size_bits, msg, HEAD);
506 
507  unsigned id_bits = 0;
508  id_codec()->field_size(&id_bits, dccl_id, nullptr);
509  head_size_bits += id_bits;
510 
511  unsigned body_size_bits;
512  codec->base_size(&body_size_bits, msg, BODY);
513 
514  const unsigned head_size_bytes = ceil_bits2bytes(head_size_bits);
515  const unsigned body_size_bytes = ceil_bits2bytes(body_size_bits);
516  return head_size_bytes + body_size_bytes;
517 }
518 
519 unsigned dccl::Codec::max_size(const google::protobuf::Descriptor* desc) const
520 {
521  std::shared_ptr<FieldCodecBase> codec = manager_.find(desc);
522 
523  unsigned head_size_bits;
524  codec->base_max_size(&head_size_bits, desc, HEAD);
525 
526  unsigned id_bits = 0;
527  id_codec()->field_max_size(&id_bits, nullptr);
528  head_size_bits += id_bits;
529 
530  unsigned body_size_bits;
531  codec->base_max_size(&body_size_bits, desc, BODY);
532 
533  const unsigned head_size_bytes = ceil_bits2bytes(head_size_bits);
534  const unsigned body_size_bytes = ceil_bits2bytes(body_size_bits);
535  return head_size_bytes + body_size_bytes;
536 }
537 
538 unsigned dccl::Codec::min_size(const google::protobuf::Descriptor* desc) const
539 {
540  std::shared_ptr<FieldCodecBase> codec = manager_.find(desc);
541 
542  unsigned head_size_bits;
543  codec->base_min_size(&head_size_bits, desc, HEAD);
544 
545  unsigned id_bits = 0;
546  id_codec()->field_min_size(&id_bits, nullptr);
547  head_size_bits += id_bits;
548 
549  unsigned body_size_bits;
550  codec->base_min_size(&body_size_bits, desc, BODY);
551 
552  const unsigned head_size_bytes = ceil_bits2bytes(head_size_bits);
553  const unsigned body_size_bytes = ceil_bits2bytes(body_size_bits);
554  return head_size_bytes + body_size_bytes;
555 }
556 
557 void dccl::Codec::info(const google::protobuf::Descriptor* desc, std::ostream* param_os /*= 0 */,
558  int user_id /* = -1 */) const
559 {
560  bool is_dlog = false;
561  if (!param_os || param_os == &dlog)
562  is_dlog = true;
563  std::stringstream ss;
564  std::ostream* os = (is_dlog) ? &ss : param_os;
565 
566  if (!is_dlog || dlog.check(INFO))
567  {
568  try
569  {
570  std::shared_ptr<FieldCodecBase> codec = manager_.find(desc);
571 
572  unsigned config_head_bit_size, body_bit_size;
573  codec->base_max_size(&config_head_bit_size, desc, HEAD);
574  codec->base_max_size(&body_bit_size, desc, BODY);
575 
576  unsigned dccl_id = (user_id < 0) ? id(desc) : user_id;
577  unsigned id_bit_size = 0;
578  id_codec()->field_size(&id_bit_size, dccl_id, nullptr);
579 
580  const unsigned bit_size = id_bit_size + config_head_bit_size + body_bit_size;
581 
582  const unsigned byte_size = ceil_bits2bytes(config_head_bit_size + id_bit_size) +
583  ceil_bits2bytes(body_bit_size);
584 
585  const unsigned allowed_byte_size = desc->options().GetExtension(dccl::msg).max_bytes();
586  const unsigned allowed_bit_size = allowed_byte_size * BITS_IN_BYTE;
587 
588  std::string message_name = std::to_string(dccl_id) + ": " + desc->full_name();
589  std::string guard = build_guard_for_console_output(message_name, '=');
590  std::string bits_dccl_head_str = "dccl.id head";
591  std::string bits_user_head_str = "user head";
592  std::string bits_body_str = "body";
593  std::string bits_padding_str = "padding to full byte";
594 
595  const int bits_width = 40;
596  const int spaces = 8;
597  std::string indent = std::string(spaces, ' ');
598 
599  *os << guard << " " << message_name << " " << guard << "\n"
600  << "Actual maximum size of message: " << byte_size << " bytes / "
601  << byte_size * BITS_IN_BYTE << " bits\n"
602  << indent << bits_dccl_head_str << std::setfill('.')
603  << std::setw(bits_width - bits_dccl_head_str.size()) << id_bit_size << "\n"
604  << indent << bits_user_head_str << std::setfill('.')
605  << std::setw(bits_width - bits_user_head_str.size()) << config_head_bit_size << "\n"
606  << indent << bits_body_str << std::setfill('.')
607  << std::setw(bits_width - bits_body_str.size()) << body_bit_size << "\n"
608  << indent << bits_padding_str << std::setfill('.')
609  << std::setw(bits_width - bits_padding_str.size())
610  << byte_size * BITS_IN_BYTE - bit_size << "\n"
611  << "Allowed maximum size of message: " << allowed_byte_size << " bytes / "
612  << allowed_bit_size << " bits\n";
613 
614  std::string header_str = "Header";
615  std::string header_guard = build_guard_for_console_output(header_str, '-');
616 
617  *os << header_guard << " " << header_str << " " << header_guard << "\n";
618  *os << bits_dccl_head_str << std::setfill('.')
619  << std::setw(bits_width - bits_dccl_head_str.size() + spaces) << id_bit_size << " {"
620  << id_codec()->name() << "}\n";
621  codec->base_info(os, desc, HEAD);
622  // *os << std::string(header_str.size() + 2 + 2*header_guard.size(), '-') << "\n";
623 
624  std::string body_str = "Body";
625  std::string body_guard = build_guard_for_console_output(body_str, '-');
626 
627  *os << body_guard << " " << body_str << " " << body_guard << "\n";
628  codec->base_info(os, desc, BODY);
629  // *os << std::string(body_str.size() + 2 + 2*body_guard.size(), '-') << "\n";
630 
631  // *os << std::string(desc->full_name().size() + 2 + 2*guard.size(), '=') << "\n";
632  os->flush();
633 
634  if (is_dlog)
635  dlog.is(INFO) && dlog << ss.str() << std::endl;
636  }
637  catch (Exception& e)
638  {
639  dlog.is(DEBUG1) &&
640  dlog << "Message " << desc->full_name()
641  << " cannot provide information due to invalid configuration. Reason: "
642  << e.what() << std::endl;
643  }
644  }
645 }
646 
647 void dccl::Codec::encrypt(std::string* s, const std::string& nonce /* message head */)
648 {
649 #if DCCL_HAS_CRYPTOPP
650  using namespace CryptoPP;
651 
652  std::string iv;
653  SHA256 hash;
654  StringSource unused(nonce, true, new HashFilter(hash, new StringSink(iv)));
655 
656  CTR_Mode<AES>::Encryption encryptor;
657  encryptor.SetKeyWithIV((byte*)crypto_key_.c_str(), crypto_key_.size(), (byte*)iv.c_str());
658 
659  std::string cipher;
660  StreamTransformationFilter in(encryptor, new StringSink(cipher));
661  in.Put((byte*)s->c_str(), s->size());
662  in.MessageEnd();
663  *s = cipher;
664 #endif
665 }
666 
667 void dccl::Codec::decrypt(std::string* s, const std::string& nonce)
668 {
669 #if DCCL_HAS_CRYPTOPP
670  using namespace CryptoPP;
671 
672  std::string iv;
673  SHA256 hash;
674  StringSource unused(nonce, true, new HashFilter(hash, new StringSink(iv)));
675 
676  CTR_Mode<AES>::Decryption decryptor;
677  decryptor.SetKeyWithIV((byte*)crypto_key_.c_str(), crypto_key_.size(), (byte*)iv.c_str());
678 
679  std::string recovered;
680 
681  StreamTransformationFilter out(decryptor, new StringSink(recovered));
682  out.Put((byte*)s->c_str(), s->size());
683  out.MessageEnd();
684  *s = recovered;
685 #endif
686 }
687 
688 void dccl::Codec::load_library(const std::string& library_path)
689 {
690  void* handle = dlopen(library_path.c_str(), RTLD_LAZY);
691  if (handle)
692  dl_handles_.push_back(handle);
693  load_library(handle);
694 }
695 
696 void dccl::Codec::load_library(void* dl_handle)
697 {
698  if (!dl_handle)
699  throw(Exception("Null shared library handle passed to load_library"));
700 
701  // load any shared library codecs
702  void (*dccl_load_ptr)(dccl::Codec*);
703  dccl_load_ptr = (void (*)(dccl::Codec*))dlsym(dl_handle, "dccl3_load");
704  if (dccl_load_ptr)
705  (*dccl_load_ptr)(this);
706 }
707 
708 void dccl::Codec::unload_library(void* dl_handle)
709 {
710  if (!dl_handle)
711  throw(Exception("Null shared library handle passed to unload_library"));
712 
713  // unload any shared library codecs
714  void (*dccl_unload_ptr)(dccl::Codec*);
715  dccl_unload_ptr = (void (*)(dccl::Codec*))dlsym(dl_handle, "dccl3_unload");
716  if (dccl_unload_ptr)
717  (*dccl_unload_ptr)(this);
718 }
719 
720 void dccl::Codec::set_crypto_passphrase(
721  const std::string& passphrase,
722  const std::set<unsigned>& do_not_encrypt_ids_ /*= std::set<unsigned>()*/)
723 {
724  if (!crypto_key_.empty())
725  crypto_key_.clear();
726  skip_crypto_ids_.clear();
727 
728 #if DCCL_HAS_CRYPTOPP
729  using namespace CryptoPP;
730 
731  SHA256 hash;
732  StringSource unused(passphrase, true, new HashFilter(hash, new StringSink(crypto_key_)));
733 
734  dlog.is(DEBUG1) && dlog << "Cryptography enabled with given passphrase" << std::endl;
735 #else
736  dlog.is(DEBUG1) && dlog << "Cryptography disabled because DCCL was compiled without support of "
737  "Crypto++. Install Crypto++ and recompile to enable cryptography."
738  << std::endl;
739 #endif
740 
741  skip_crypto_ids_ = do_not_encrypt_ids_;
742 }
743 
744 void dccl::Codec::info_all(std::ostream* param_os /*= 0 */) const
745 {
746  bool is_dlog = false;
747  if (!param_os || param_os == &dlog)
748  is_dlog = true;
749  std::stringstream ss;
750  std::ostream* os = (is_dlog) ? &ss : param_os;
751  if (!is_dlog || dlog.check(INFO))
752  {
753  std::string codec_str = "Dynamic Compact Control Language (DCCL) Codec";
754  std::string codec_guard = build_guard_for_console_output(codec_str, '|');
755 
756  *os << codec_guard << " " << codec_str << " " << codec_guard << "\n";
757  *os << id2desc_.size() << " messages loaded.\n";
758  *os << "Field sizes are in bits unless otherwise noted."
759  << "\n";
760 
761  for (auto it : id2desc_) info(it.second, os, it.first);
762  os->flush();
763 
764  if (is_dlog)
765  dlog.is(INFO) && dlog << ss.str() << std::endl;
766  }
767 }
768 
769 void dccl::Codec::set_id_codec(const std::string& id_codec_name)
770 {
771  // we must reload messages after setting the id_codec
772  unload_all();
773 
774  id_codec_ = id_codec_name;
775  // make sure the id codec exists
776  id_codec();
777 }
778 
779 std::string dccl::Codec::build_guard_for_console_output(std::string& base, char guard_char) const
780 {
781  // Only guard if possible, otherwise return an empty string rather than throwing a std::length_error.
782  return (base.size() < console_width_)
783  ? std::string((console_width_ - base.size()) / 2, guard_char)
784  : std::string();
785 }
786 
787 std::string dccl::Codec::get_all_error_fields_in_message(const google::protobuf::Message& message,
788  uint8_t depth /*= 1 */)
789 {
790  // This is largely taken from google::protobuf::ReflectionOps::FindInitializationErrors(), which is called by
791  // google::protobuf::Message::FindInitializationErrors(). The Message implementation is not used as they force a
792  // prefix of parent message onto fields, which would require a split function to break by delimiter '.' should we
793  // want to reflect upon sub-messages and get field numbers.
794 
795  std::stringstream output_stream;
796 
797  const google::protobuf::Descriptor* descriptor = message.GetDescriptor();
798  const google::protobuf::Reflection* reflection = message.GetReflection();
799  const uint8_t depth_spacing = 4;
800 
801  // Check required fields of this message.
802  const int32_t field_count = descriptor->field_count();
803 
804  for (int32_t index = 0; index < field_count; ++index)
805  {
806  if (descriptor->field(index)->is_required())
807  {
808  if (!reflection->HasField(message, descriptor->field(index)))
809  {
810  output_stream << std::string(depth * depth_spacing, ' ')
811  << descriptor->field(index)->number() << ": "
812  << descriptor->field(index)->name() << "\n";
813  }
814  }
815  }
816 
817  // Check sub-messages.
818  std::vector<const google::protobuf::FieldDescriptor*> fields;
819  reflection->ListFields(message, &fields);
820 
821  for (const google::protobuf::FieldDescriptor* field : fields)
822  {
823  if (field->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE)
824  {
825  output_stream << std::string(depth * depth_spacing, ' ') << field->number() << ": "
826  << field->name() << "\n";
827 
828  if (field->is_repeated())
829  {
830  int32_t size = reflection->FieldSize(message, field);
831 
832  for (int32_t index = 0; index < size; ++index)
833  {
834  const google::protobuf::Message& sub_message =
835  reflection->GetRepeatedMessage(message, field, index);
836 
837  output_stream << std::string((depth + 1) * depth_spacing, ' ') << "[" << index
838  << "]: "
839  << "\n";
840 
841  output_stream << get_all_error_fields_in_message(sub_message, depth + 2);
842  }
843  }
844  else
845  {
846  const google::protobuf::Message& sub_message =
847  reflection->GetMessage(message, field);
848  output_stream << get_all_error_fields_in_message(sub_message, depth + 1);
849  }
850  }
851  }
852 
853  return output_stream.str();
854 }
dccl::v2::StaticCodec
Placeholder codec that takes no space on the wire (0 bits).
Definition: field_codec_default.h:403
dccl::v2::TimeCodec< uint64 >
Definition: field_codec_default.h:392
dccl::v4::DefaultMessageCodec
Provides the default codec for encoding a base Google Protobuf message or an embedded message by call...
Definition: field_codec_default_message.h:40
dccl
Dynamic Compact Control Language namespace.
Definition: any.h:49
dccl::Logger::is
bool is(logger::Verbosity verbosity, logger::Group group=logger::GENERAL)
Indicates the verbosity of the Logger until the next std::flush or std::endl. The boolean return is u...
Definition: logger.h:191
dccl::Exception
Exception class for DCCL.
Definition: exception.h:47
dccl::v2::TimeCodec< int64 >
Definition: field_codec_default.h:395
dccl::Codec::unload
void unload()
Unload a given message.
Definition: codec.h:124
dccl::Codec::set_id_codec
void set_id_codec(const std::string &id_codec_name)
Set a different ID codec name (note that is calls unload_all() so all messages must be reloaded)
Definition: codec.cpp:769
dccl::Codec::min_size
unsigned min_size()
Provides the encoded minimum size (in bytes) of msg.
Definition: codec.h:352
dccl::Codec::max_size
unsigned max_size()
Provides the encoded maximum size (in bytes) of msg.
Definition: codec.h:340
dccl::internal::MessageStack
Definition: field_codec_message_stack.h:72
dccl::Codec
The Dynamic CCL enCODer/DECoder. This is the main class you will use to load, encode and decode DCCL ...
Definition: codec.h:61
dccl::Codec::Codec
Codec(std::string dccl_id_codec_name=default_id_codec_name(), const std::string &library_path="")
Instantiate a Codec, optionally with a non-default identifier field codec (loaded via a shared librar...
Definition: codec.cpp:74
dccl::Codec::size
unsigned size(const google::protobuf::Message &msg, int user_id=-1)
Provides the encoded size (in bytes) of msg. This is useful if you need to know the size of a message...
Definition: codec.cpp:497
dccl::v3::DefaultStringCodec
Provides an variable length ASCII string encoder.
Definition: field_codec_default.h:89
dccl::v3::DefaultNumericFieldCodec< double >
dccl::v2::DefaultStringCodec
Provides an variable length ASCII string encoder. Can encode strings up to 255 bytes by using a lengt...
Definition: field_codec_default.h:268
dccl::v2::TimeCodec< double >
Definition: field_codec_default.h:398
dccl::Codec::unload_library
void unload_library(void *dl_handle)
Remove codecs and/or unload messages present in the given shared library handle.
Definition: codec.cpp:708
dccl::hex_encode
void hex_encode(CharIterator begin, CharIterator end, std::string *out, bool upper_case=false)
Encodes a (little-endian) hexadecimal string from a byte string. Index 0 of begin is written to index...
Definition: binary.h:100
dccl::v3::VarBytesCodec
Definition: field_codec_var_bytes.h:36
dccl::Bitset
A variable size container of bits (subclassed from std::deque<bool>) with an optional hierarchy....
Definition: bitset.h:41
dccl::v2::DefaultBoolCodec
Provides a bool encoder. Uses 1 bit if field is required, 2 bits if optional
Definition: field_codec_default.h:255
dccl::OutOfRangeException
Definition: exception.h:68
dccl::Codec::load_library
void load_library(void *dl_handle)
Add codecs and/or load messages present in the given shared library handle.
Definition: codec.cpp:696
dccl::v2::DefaultMessageCodec
Provides the default codec for encoding a base Google Protobuf message or an embedded message by call...
Definition: field_codec_default_message.h:38
dccl::DefaultIdentifierCodec
Provides the default 1 byte or 2 byte DCCL ID codec.
Definition: field_codec_id.h:29
dccl::Logger::check
bool check(logger::Verbosity verbosity)
Same as is() but doesn't set the verbosity or lock the mutex.
Definition: logger.h:180
dccl::FieldCodecManagerLocal::add
std::enable_if< std::is_base_of< google::protobuf::Message, typename Codec::wire_type >::value &&!std::is_same< google::protobuf::Message, typename Codec::wire_type >::value, void >::type add(const std::string &name)
Add a new field codec (used for codecs operating on statically generated Protobuf messages,...
Definition: field_codec_manager.h:298
dccl::Codec::encode
void encode(std::string *bytes, const google::protobuf::Message &msg, bool header_only=false, int user_id=-1)
Encodes a DCCL message.
Definition: codec.cpp:318
dccl::v2::DefaultEnumCodec
Provides an enum encoder. This converts the enumeration to an integer (based on the enumeration index...
Definition: field_codec_default.h:302
dccl::Codec::set_crypto_passphrase
void set_crypto_passphrase(const std::string &passphrase, const std::set< unsigned > &do_not_encrypt_ids_=std::set< unsigned >())
Set a passphrase to be used when encoded messages to encrypt them and to decrypt messages after decod...
Definition: codec.cpp:720
dccl::v3::DefaultEnumCodec
Provides an enum encoder. This converts the enumeration to an integer and uses DefaultNumericFieldCod...
Definition: field_codec_default.h:54
dccl::v2::DefaultBytesCodec
Provides an fixed length byte string encoder.
Definition: field_codec_default.h:288
Message
dccl::Codec::info_all
void info_all(std::ostream *os=nullptr) const
Writes a human readable summary (including field sizes) of all the loaded (validated) DCCL types.
Definition: codec.cpp:744
dccl::Codec::load
void load()
All messages must be explicited loaded and validated (size checks, option extensions checks,...
Definition: codec.h:119
dccl::Codec::info
void info(std::ostream *os=nullptr, int user_id=-1) const
Writes a human readable summary (including field sizes) of the provided DCCL type to the stream provi...
Definition: codec.h:185
dccl::v3::PresenceBitCodec
Encodes empty optional fields with a single "presence" bit.
Definition: field_codec_presence.h:43
dccl::Codec::decode
CharIterator decode(CharIterator begin, CharIterator end, google::protobuf::Message *msg, bool header_only=false)
Decode a DCCL message when the type is known at compile time.
Definition: codec.h:484
dccl::v2::DefaultNumericFieldCodec
Provides a basic bounded arbitrary length numeric (double, float, uint32, uint64, int32,...
Definition: field_codec_default.h:53
dccl::Codec::id
unsigned id() const
Gives the DCCL id (defined by the custom message option extension "(dccl.msg).id" in the ....
Definition: codec.h:206
dccl::v3::DefaultMessageCodec
Provides the default codec for encoding a base Google Protobuf message or an embedded message by call...
Definition: field_codec_default_message.h:38
dccl::Codec::~Codec
virtual ~Codec()
Destructor.
Definition: codec.cpp:86
dccl::Bitset::to_byte_string
std::string to_byte_string()
Returns the value of the Bitset to a byte string, where each character represents 8 bits of the Bitse...
Definition: bitset.h:296