DCCL v4
codec.cpp
1 // Copyright 2009-2017 Toby Schneider (http://gobysoft.org/index.wt/people/toby)
2 // GobySoft, LLC (for 2013-)
3 // Massachusetts Institute of Technology (for 2007-2014)
4 // Community contributors (see AUTHORS file)
5 //
6 //
7 // This file is part of the Dynamic Compact Control Language Library
8 // ("DCCL").
9 //
10 // DCCL is free software: you can redistribute it and/or modify
11 // it under the terms of the GNU Lesser General Public License as published by
12 // the Free Software Foundation, either version 2.1 of the License, or
13 // (at your option) any later version.
14 //
15 // DCCL is distributed in the hope that it will be useful,
16 // but WITHOUT ANY WARRANTY; without even the implied warranty of
17 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 // GNU Lesser General Public License for more details.
19 //
20 // You should have received a copy of the GNU Lesser General Public License
21 // along with DCCL. If not, see <http://www.gnu.org/licenses/>.
22 #include <algorithm>
23 
24 #include <dlfcn.h> // for shared library loading
25 
26 #include "dccl/codec.h"
27 
28 #if DCCL_HAS_CRYPTOPP
29 #if CRYPTOPP_PATH_USES_PLUS_SIGN
30 #include <crypto++/aes.h>
31 #include <crypto++/filters.h>
32 #include <crypto++/modes.h>
33 #include <crypto++/sha.h>
34 #else
35 #include <cryptopp/aes.h>
36 #include <cryptopp/filters.h>
37 #include <cryptopp/modes.h>
38 #include <cryptopp/sha.h>
39 #endif // CRYPTOPP_PATH_USES_PLUS_SIGN
40 #endif // HAS_CRYPTOPP
41 
42 #include "dccl/codecs2/field_codec_default.h"
43 #include "dccl/codecs3/field_codec_default.h"
44 #include "dccl/codecs3/field_codec_presence.h"
45 #include "dccl/codecs3/field_codec_var_bytes.h"
46 #include "dccl/codecs4/field_codec_default.h"
47 #include "dccl/codecs4/field_codec_default_message.h"
48 #include "dccl/field_codec_id.h"
49 
50 #include "dccl/option_extensions.pb.h"
51 
52 using dccl::hex_decode;
53 using dccl::hex_encode;
54 
55 using namespace dccl;
56 using namespace dccl::logger;
57 
58 using google::protobuf::Descriptor;
59 using google::protobuf::FieldDescriptor;
60 using google::protobuf::Reflection;
61 
62 //
63 // Codec
64 //
65 
66 dccl::Codec::Codec(const std::string& dccl_id_codec, const std::string& library_path)
67  : strict_(false), id_codec_(dccl_id_codec), console_width_(60)
68 {
69  set_default_codecs();
70  FieldCodecManager::add<DefaultIdentifierCodec>(default_id_codec_name());
71 
72  if (!library_path.empty())
73  load_library(library_path);
74  // make sure the id codec exists
75  id_codec();
76 }
77 
78 dccl::Codec::~Codec()
79 {
80  for (std::vector<void*>::iterator it = dl_handles_.begin(), n = dl_handles_.end(); it != n;
81  ++it)
82  {
83  unload_library(*it);
84  dlclose(*it);
85  }
86 }
87 
88 void dccl::Codec::set_default_codecs()
89 {
90  // only need to load these once into the static FieldCodecManager
91  static bool defaults_loaded = false;
92 
93  if (!defaults_loaded)
94  {
95  using google::protobuf::FieldDescriptor;
96 
97  // version 2
98  FieldCodecManager::add<v2::DefaultNumericFieldCodec<double>>(default_codec_name());
99  FieldCodecManager::add<v2::DefaultNumericFieldCodec<float>>(default_codec_name());
100  FieldCodecManager::add<v2::DefaultBoolCodec>(default_codec_name());
101  FieldCodecManager::add<v2::DefaultNumericFieldCodec<int32>>(default_codec_name());
102  FieldCodecManager::add<v2::DefaultNumericFieldCodec<int64>>(default_codec_name());
103  FieldCodecManager::add<v2::DefaultNumericFieldCodec<uint32>>(default_codec_name());
104  FieldCodecManager::add<v2::DefaultNumericFieldCodec<uint64>>(default_codec_name());
105  FieldCodecManager::add<v2::DefaultStringCodec, FieldDescriptor::TYPE_STRING>(
106  default_codec_name());
107  FieldCodecManager::add<v2::DefaultBytesCodec, FieldDescriptor::TYPE_BYTES>(
108  default_codec_name());
109  FieldCodecManager::add<v2::DefaultEnumCodec>(default_codec_name());
110  FieldCodecManager::add<v2::DefaultMessageCodec, FieldDescriptor::TYPE_MESSAGE>(
111  default_codec_name());
112 
113  FieldCodecManager::add<v2::TimeCodec<uint64>>("dccl.time2");
114  FieldCodecManager::add<v2::TimeCodec<int64>>("dccl.time2");
115  FieldCodecManager::add<v2::TimeCodec<double>>("dccl.time2");
116 
117  FieldCodecManager::add<v2::StaticCodec<std::string>>("dccl.static2");
118  FieldCodecManager::add<v2::StaticCodec<double>>("dccl.static2");
119  FieldCodecManager::add<v2::StaticCodec<float>>("dccl.static2");
120  FieldCodecManager::add<v2::StaticCodec<int32>>("dccl.static2");
121  FieldCodecManager::add<v2::StaticCodec<int64>>("dccl.static2");
122  FieldCodecManager::add<v2::StaticCodec<uint32>>("dccl.static2");
123  FieldCodecManager::add<v2::StaticCodec<uint64>>("dccl.static2");
124 
125  // version 3
126  FieldCodecManager::add<v3::DefaultNumericFieldCodec<double>>(default_codec_name(3));
127  FieldCodecManager::add<v3::DefaultNumericFieldCodec<float>>(default_codec_name(3));
128  FieldCodecManager::add<v3::DefaultBoolCodec>(default_codec_name(3));
129  FieldCodecManager::add<v3::DefaultNumericFieldCodec<int32>>(default_codec_name(3));
130  FieldCodecManager::add<v3::DefaultNumericFieldCodec<int64>>(default_codec_name(3));
131  FieldCodecManager::add<v3::DefaultNumericFieldCodec<uint32>>(default_codec_name(3));
132  FieldCodecManager::add<v3::DefaultNumericFieldCodec<uint64>>(default_codec_name(3));
133  FieldCodecManager::add<v3::DefaultStringCodec, FieldDescriptor::TYPE_STRING>(
134  default_codec_name(3));
135  FieldCodecManager::add<v3::DefaultBytesCodec, FieldDescriptor::TYPE_BYTES>(
136  default_codec_name(3));
137  FieldCodecManager::add<v3::DefaultEnumCodec>(default_codec_name(3));
138  FieldCodecManager::add<v3::DefaultMessageCodec, FieldDescriptor::TYPE_MESSAGE>(
139  default_codec_name(3));
140 
141  // version 4
142  FieldCodecManager::add<v4::DefaultNumericFieldCodec<double>>(default_codec_name(4));
143  FieldCodecManager::add<v4::DefaultNumericFieldCodec<float>>(default_codec_name(4));
144  FieldCodecManager::add<v4::DefaultBoolCodec>(default_codec_name(4));
145  FieldCodecManager::add<v4::DefaultNumericFieldCodec<int32>>(default_codec_name(4));
146  FieldCodecManager::add<v4::DefaultNumericFieldCodec<int64>>(default_codec_name(4));
147  FieldCodecManager::add<v4::DefaultNumericFieldCodec<uint32>>(default_codec_name(4));
148  FieldCodecManager::add<v4::DefaultNumericFieldCodec<uint64>>(default_codec_name(4));
149  FieldCodecManager::add<v4::DefaultStringCodec, FieldDescriptor::TYPE_STRING>(
150  default_codec_name(4));
151  FieldCodecManager::add<v4::DefaultBytesCodec, FieldDescriptor::TYPE_BYTES>(
152  default_codec_name(4));
153  FieldCodecManager::add<v4::DefaultEnumCodec>(default_codec_name(4));
154  FieldCodecManager::add<v4::DefaultMessageCodec, FieldDescriptor::TYPE_MESSAGE>(
155  default_codec_name(4));
156 
157  // presence bit codec, which encode empty optional fields with a single bit
158  FieldCodecManager::add<v3::PresenceBitCodec<v3::DefaultNumericFieldCodec<double>>>(
159  "dccl.presence");
160  FieldCodecManager::add<v3::PresenceBitCodec<v3::DefaultNumericFieldCodec<float>>>(
161  "dccl.presence");
162  FieldCodecManager::add<v3::PresenceBitCodec<v3::DefaultNumericFieldCodec<int32>>>(
163  "dccl.presence");
164  FieldCodecManager::add<v3::PresenceBitCodec<v3::DefaultNumericFieldCodec<int64>>>(
165  "dccl.presence");
166  FieldCodecManager::add<v3::PresenceBitCodec<v3::DefaultNumericFieldCodec<uint32>>>(
167  "dccl.presence");
168  FieldCodecManager::add<v3::PresenceBitCodec<v3::DefaultNumericFieldCodec<uint64>>>(
169  "dccl.presence");
170  FieldCodecManager::add<v3::PresenceBitCodec<v3::DefaultEnumCodec>>("dccl.presence");
171 
172  // alternative bytes codec that more efficiently encodes variable length bytes fields
173  FieldCodecManager::add<v3::VarBytesCodec, FieldDescriptor::TYPE_BYTES>("dccl.var_bytes");
174 
175  // for backwards compatibility
176  FieldCodecManager::add<v2::TimeCodec<uint64>>("_time");
177  FieldCodecManager::add<v2::TimeCodec<int64>>("_time");
178  FieldCodecManager::add<v2::TimeCodec<double>>("_time");
179 
180  FieldCodecManager::add<v2::StaticCodec<std::string>>("_static");
181  FieldCodecManager::add<v2::StaticCodec<double>>("_static");
182  FieldCodecManager::add<v2::StaticCodec<float>>("_static");
183  FieldCodecManager::add<v2::StaticCodec<int32>>("_static");
184  FieldCodecManager::add<v2::StaticCodec<int64>>("_static");
185  FieldCodecManager::add<v2::StaticCodec<uint32>>("_static");
186  FieldCodecManager::add<v2::StaticCodec<uint64>>("_static");
187 
188  defaults_loaded = true;
189  }
190 }
191 
192 void dccl::Codec::encode_internal(const google::protobuf::Message& msg, bool header_only,
193  Bitset& head_bits, Bitset& body_bits, int user_id)
194 {
195  const Descriptor* desc = msg.GetDescriptor();
196 
197  dlog.is(DEBUG1, ENCODE) && dlog << "Began encoding message of type: " << desc->full_name()
198  << std::endl;
199 
200  try
201  {
202  unsigned dccl_id = (user_id < 0) ? id(desc) : user_id;
203  size_t head_byte_size = 0;
204 
205  if (!msg.IsInitialized() && !header_only)
206  {
207  std::stringstream ss;
208 
209  ss << "Message is not properly initialized. All `required` fields must be set. Fields "
210  "with errors: \n"
211  << get_all_error_fields_in_message(msg);
212 
213  throw(Exception(ss.str()));
214  }
215 
216  if (!id2desc_.count(dccl_id))
217  throw(Exception("Message id " + boost::lexical_cast<std::string>(dccl_id) +
218  " has not been loaded. Call load() before encoding this type."));
219 
220  boost::shared_ptr<FieldCodecBase> codec = FieldCodecManager::find(desc);
221  boost::shared_ptr<internal::FromProtoCppTypeBase> helper = internal::TypeHelper::find(desc);
222 
223  if (codec)
224  {
225  //fixed header
226  id_codec()->field_encode(&head_bits, dccl_id, 0);
227 
228  internal::MessageStack msg_stack;
229  msg_stack.push(msg.GetDescriptor());
230  codec->base_encode(&head_bits, msg, HEAD, strict_);
231 
232  // given header of not even byte size (e.g. 01011), make even byte size (e.g. 00001011)
233  head_byte_size = ceil_bits2bytes(head_bits.size());
234  head_bits.resize(head_byte_size * BITS_IN_BYTE);
235 
236  if (header_only)
237  {
238  dlog.is(DEBUG2, ENCODE) &&
239  dlog << "as requested, skipping encoding and encrypting body." << std::endl;
240  }
241  else
242  {
243  codec->base_encode(&body_bits, msg, BODY, strict_);
244  }
245  }
246  else
247  {
248  throw(Exception("Failed to find (dccl.msg).codec `" +
249  desc->options().GetExtension(dccl::msg).codec() + "`"));
250  }
251  }
252  catch (dccl::OutOfRangeException& e)
253  {
254  dlog.is(DEBUG1, ENCODE) &&
255  dlog << "Message " << desc->full_name()
256  << " failed to encode because a field was out of bounds and strict == true: "
257  << e.what() << std::endl;
258  throw;
259  }
260  catch (std::exception& e)
261  {
262  std::stringstream ss;
263 
264  ss << "Message " << desc->full_name() << " failed to encode. Reason: " << e.what();
265 
266  dlog.is(DEBUG1, ENCODE) && dlog << ss.str() << std::endl;
267  throw(Exception(ss.str()));
268  }
269 }
270 
271 size_t dccl::Codec::encode(char* bytes, size_t max_len, const google::protobuf::Message& msg,
272  bool header_only /* = false */, int user_id /* = -1 */)
273 {
274  const Descriptor* desc = msg.GetDescriptor();
275  Bitset head_bits;
276  Bitset body_bits;
277  encode_internal(msg, header_only, head_bits, body_bits, user_id);
278 
279  size_t head_byte_size = ceil_bits2bytes(head_bits.size());
280  if (max_len < head_byte_size)
281  {
282  throw std::length_error("max_len must be >= head_byte_size");
283  }
284  head_bits.to_byte_string(bytes, head_byte_size);
285 
286  dlog.is(DEBUG2, ENCODE) && dlog << "Head bytes (bits): " << head_byte_size << "("
287  << head_bits.size() << ")" << std::endl;
288  dlog.is(DEBUG3, ENCODE) && dlog << "Unencrypted Head (bin): " << head_bits << std::endl;
289  dlog.is(DEBUG3, ENCODE) && dlog << "Unencrypted Head (hex): "
290  << hex_encode(bytes, bytes + head_byte_size) << std::endl;
291 
292  size_t body_byte_size = 0;
293  if (!header_only)
294  {
295  body_byte_size = ceil_bits2bytes(body_bits.size());
296  if (max_len < (head_byte_size + body_byte_size))
297  {
298  throw std::length_error("max_len must be >= (head_byte_size + body_byte_size)");
299  }
300  body_bits.to_byte_string(bytes + head_byte_size, max_len - head_byte_size);
301 
302  dlog.is(DEBUG3, ENCODE) && dlog << "Unencrypted Body (bin): " << body_bits << std::endl;
303  dlog.is(DEBUG3, ENCODE) &&
304  dlog << "Unencrypted Body (hex): "
305  << hex_encode(bytes + head_byte_size, bytes + head_byte_size + body_byte_size)
306  << std::endl;
307  dlog.is(DEBUG2, ENCODE) && dlog << "Body bytes (bits): " << body_byte_size << "("
308  << body_bits.size() << ")" << std::endl;
309 
310  unsigned dccl_id = (user_id < 0) ? id(desc) : user_id;
311  if (!crypto_key_.empty() && !skip_crypto_ids_.count(dccl_id))
312  {
313  std::string head_bytes(bytes, bytes + head_byte_size);
314  std::string body_bytes(bytes + head_byte_size, bytes + head_byte_size + body_byte_size);
315  encrypt(&body_bytes, head_bytes);
316  std::memcpy(bytes + head_byte_size, body_bytes.data(), body_bytes.size());
317  }
318 
319  dlog.is(logger::DEBUG3, logger::ENCODE) &&
320  dlog << "Encrypted Body (hex): "
321  << hex_encode(bytes + head_byte_size, bytes + head_byte_size + body_byte_size)
322  << std::endl;
323  }
324 
325  dlog.is(DEBUG1, ENCODE) && dlog << "Successfully encoded message of type: " << desc->full_name()
326  << std::endl;
327 
328  return head_byte_size + body_byte_size;
329 }
330 
331 void dccl::Codec::encode(std::string* bytes, const google::protobuf::Message& msg,
332  bool header_only /* = false */, int user_id /* = -1 */)
333 {
334  const Descriptor* desc = msg.GetDescriptor();
335  Bitset head_bits;
336  Bitset body_bits;
337  encode_internal(msg, header_only, head_bits, body_bits, user_id);
338 
339  std::string head_bytes = head_bits.to_byte_string();
340 
341  dlog.is(DEBUG2, ENCODE) && dlog << "Head bytes (bits): " << head_bytes.size() << "("
342  << head_bits.size() << ")" << std::endl;
343  dlog.is(DEBUG3, ENCODE) && dlog << "Unencrypted Head (bin): " << head_bits << std::endl;
344  dlog.is(DEBUG3, ENCODE) && dlog << "Unencrypted Head (hex): " << hex_encode(head_bytes)
345  << std::endl;
346 
347  std::string body_bytes;
348  if (!header_only)
349  {
350  body_bytes = body_bits.to_byte_string();
351 
352  dlog.is(DEBUG3, ENCODE) && dlog << "Unencrypted Body (bin): " << body_bits << std::endl;
353  dlog.is(DEBUG3, ENCODE) && dlog << "Unencrypted Body (hex): " << hex_encode(body_bytes)
354  << std::endl;
355  dlog.is(DEBUG2, ENCODE) && dlog << "Body bytes (bits): " << body_bytes.size() << "("
356  << body_bits.size() << ")" << std::endl;
357 
358  unsigned dccl_id = (user_id < 0) ? id(desc) : user_id;
359  if (!crypto_key_.empty() && !skip_crypto_ids_.count(dccl_id))
360  encrypt(&body_bytes, head_bytes);
361 
362  dlog.is(logger::DEBUG3, logger::ENCODE) &&
363  dlog << "Encrypted Body (hex): " << hex_encode(body_bytes) << std::endl;
364  }
365 
366  dlog.is(DEBUG1, ENCODE) && dlog << "Successfully encoded message of type: " << desc->full_name()
367  << std::endl;
368  *bytes += head_bytes + body_bytes;
369 }
370 
371 unsigned dccl::Codec::id(const std::string& bytes) const { return id(bytes.begin(), bytes.end()); }
372 
373 void dccl::Codec::decode(std::string* bytes, google::protobuf::Message* msg)
374 {
375  decode(*bytes, msg);
376  unsigned last_size = size(*msg);
377  bytes->erase(0, last_size);
378 }
379 
380 void dccl::Codec::decode(const std::string& bytes, google::protobuf::Message* msg,
381  bool header_only /* = false */)
382 {
383  decode(bytes.begin(), bytes.end(), msg, header_only);
384 }
385 
386 // makes sure we can actual encode / decode a message of this descriptor given the loaded FieldCodecs
387 // checks all bounds on the message
388 void dccl::Codec::load(const google::protobuf::Descriptor* desc, int user_id /* = -1 */)
389 {
390  try
391  {
392  if (user_id < 0 && !desc->options().GetExtension(dccl::msg).has_id())
393  throw(
394  Exception("Missing message option `(dccl.msg).id`. Specify a unique id (e.g. 3) in "
395  "the body of your .proto message using \"option (dccl.msg).id = 3\""));
396  if (!desc->options().GetExtension(dccl::msg).has_max_bytes())
397  throw(Exception("Missing message option `(dccl.msg).max_bytes`. Specify a maximum "
398  "(encoded) message size in bytes (e.g. 32) in the body of your .proto "
399  "message using \"option (dccl.msg).max_bytes = 32\""));
400 
401  if (!desc->options().GetExtension(dccl::msg).has_codec_version())
402  throw(Exception("No (dccl.msg).codec_version set for DCCL Message '" +
403  desc->full_name() +
404  "'. For new messages, set 'option (dccl.msg).codec_version = 4' in the "
405  "message definition for " +
406  desc->full_name() + " to use the default DCCL4 codecs."));
407 
408  boost::shared_ptr<FieldCodecBase> codec = FieldCodecManager::find(desc);
409 
410  unsigned dccl_id = (user_id < 0) ? id(desc) : user_id;
411  unsigned head_size_bits, body_size_bits;
412  codec->base_max_size(&head_size_bits, desc, HEAD);
413  codec->base_max_size(&body_size_bits, desc, BODY);
414 
415  unsigned id_bits = 0;
416  id_codec()->field_size(&id_bits, dccl_id, 0);
417  head_size_bits += id_bits;
418 
419  const unsigned byte_size =
420  ceil_bits2bytes(head_size_bits) + ceil_bits2bytes(body_size_bits);
421 
422  if (byte_size > desc->options().GetExtension(dccl::msg).max_bytes())
423  throw(Exception(
424  "Actual maximum size of message exceeds allowed maximum (dccl.max_bytes). Tighten "
425  "bounds, remove fields, improve codecs, or increase the allowed dccl.max_bytes"));
426 
427  codec->base_validate(desc, HEAD);
428  codec->base_validate(desc, BODY);
429 
430  if (id2desc_.count(dccl_id) && desc != id2desc_.find(dccl_id)->second)
431  throw(Exception("`dccl id` " + boost::lexical_cast<std::string>(dccl_id) +
432  " is already in use by Message " +
433  id2desc_.find(dccl_id)->second->full_name() + ": " +
434  boost::lexical_cast<std::string>(id2desc_.find(dccl_id)->second)));
435  else
436  id2desc_.insert(std::make_pair(dccl_id, desc));
437 
438  dlog.is(DEBUG1) && dlog << "Successfully validated message of type: " << desc->full_name()
439  << std::endl;
440  }
441  catch (Exception& e)
442  {
443  try
444  {
445  info(desc, &dlog);
446  }
447  catch (Exception& e)
448  {
449  }
450 
451  dlog.is(DEBUG1) && dlog << "Message " << desc->full_name() << ": " << desc
452  << " failed validation. Reason: " << e.what() << "\n"
453  << "If possible, information about the Message are printed above. "
454  << std::endl;
455 
456  throw;
457  }
458 }
459 
460 void dccl::Codec::unload(const google::protobuf::Descriptor* desc)
461 {
462  unsigned int erased = 0;
463  for (std::map<int32, const google::protobuf::Descriptor*>::iterator it = id2desc_.begin();
464  it != id2desc_.end();)
465  {
466  if (it->second == desc)
467  {
468  erased++;
469  id2desc_.erase(it++);
470  }
471  else
472  {
473  it++;
474  }
475  }
476  if (erased == 0)
477  {
478  dlog.is(DEBUG1) && dlog << "Message " << desc->full_name()
479  << ": is not loaded. Ignoring unload request." << std::endl;
480  }
481 }
482 
483 void dccl::Codec::unload(size_t dccl_id)
484 {
485  if (id2desc_.count(dccl_id))
486  {
487  id2desc_.erase(dccl_id);
488  }
489  else
490  {
491  dlog.is(DEBUG1) && dlog << "Message with id " << dccl_id
492  << ": is not loaded. Ignoring unload request." << std::endl;
493  return;
494  }
495 }
496 
497 unsigned dccl::Codec::size(const google::protobuf::Message& msg, int user_id /* = -1 */)
498 {
499  const Descriptor* desc = msg.GetDescriptor();
500 
501  boost::shared_ptr<FieldCodecBase> codec = FieldCodecManager::find(desc);
502 
503  unsigned dccl_id = (user_id < 0) ? id(desc) : user_id;
504  unsigned head_size_bits;
505  codec->base_size(&head_size_bits, msg, HEAD);
506 
507  unsigned id_bits = 0;
508  id_codec()->field_size(&id_bits, dccl_id, 0);
509  head_size_bits += id_bits;
510 
511  unsigned body_size_bits;
512  codec->base_size(&body_size_bits, msg, BODY);
513 
514  const unsigned head_size_bytes = ceil_bits2bytes(head_size_bits);
515  const unsigned body_size_bytes = ceil_bits2bytes(body_size_bits);
516  return head_size_bytes + body_size_bytes;
517 }
518 
519 unsigned dccl::Codec::max_size(const google::protobuf::Descriptor* desc) const
520 {
521  boost::shared_ptr<FieldCodecBase> codec = FieldCodecManager::find(desc);
522 
523  unsigned head_size_bits;
524  codec->base_max_size(&head_size_bits, desc, HEAD);
525 
526  unsigned id_bits = 0;
527  id_codec()->field_max_size(&id_bits, 0);
528  head_size_bits += id_bits;
529 
530  unsigned body_size_bits;
531  codec->base_max_size(&body_size_bits, desc, BODY);
532 
533  const unsigned head_size_bytes = ceil_bits2bytes(head_size_bits);
534  const unsigned body_size_bytes = ceil_bits2bytes(body_size_bits);
535  return head_size_bytes + body_size_bytes;
536 }
537 
538 unsigned dccl::Codec::min_size(const google::protobuf::Descriptor* desc) const
539 {
540  boost::shared_ptr<FieldCodecBase> codec = FieldCodecManager::find(desc);
541 
542  unsigned head_size_bits;
543  codec->base_min_size(&head_size_bits, desc, HEAD);
544 
545  unsigned id_bits = 0;
546  id_codec()->field_min_size(&id_bits, 0);
547  head_size_bits += id_bits;
548 
549  unsigned body_size_bits;
550  codec->base_min_size(&body_size_bits, desc, BODY);
551 
552  const unsigned head_size_bytes = ceil_bits2bytes(head_size_bits);
553  const unsigned body_size_bytes = ceil_bits2bytes(body_size_bits);
554  return head_size_bytes + body_size_bytes;
555 }
556 
557 void dccl::Codec::info(const google::protobuf::Descriptor* desc, std::ostream* param_os /*= 0 */,
558  int user_id /* = -1 */) const
559 {
560  std::ostream* os = (param_os) ? param_os : &dlog;
561 
562  if (param_os || dlog.is(INFO))
563  {
564  try
565  {
566  boost::shared_ptr<FieldCodecBase> codec = FieldCodecManager::find(desc);
567 
568  unsigned config_head_bit_size, body_bit_size;
569  codec->base_max_size(&config_head_bit_size, desc, HEAD);
570  codec->base_max_size(&body_bit_size, desc, BODY);
571 
572  unsigned dccl_id = (user_id < 0) ? id(desc) : user_id;
573  unsigned id_bit_size = 0;
574  id_codec()->field_size(&id_bit_size, dccl_id, 0);
575 
576  const unsigned bit_size = id_bit_size + config_head_bit_size + body_bit_size;
577 
578  const unsigned byte_size = ceil_bits2bytes(config_head_bit_size + id_bit_size) +
579  ceil_bits2bytes(body_bit_size);
580 
581  const unsigned allowed_byte_size = desc->options().GetExtension(dccl::msg).max_bytes();
582  const unsigned allowed_bit_size = allowed_byte_size * BITS_IN_BYTE;
583 
584  std::string message_name =
585  boost::lexical_cast<std::string>(dccl_id) + ": " + desc->full_name();
586  std::string guard = build_guard_for_console_output(message_name, '=');
587  std::string bits_dccl_head_str = "dccl.id head";
588  std::string bits_user_head_str = "user head";
589  std::string bits_body_str = "body";
590  std::string bits_padding_str = "padding to full byte";
591 
592  const int bits_width = 40;
593  const int spaces = 8;
594  std::string indent = std::string(spaces, ' ');
595 
596  *os << guard << " " << message_name << " " << guard << "\n"
597  << "Actual maximum size of message: " << byte_size << " bytes / "
598  << byte_size * BITS_IN_BYTE << " bits\n"
599  << indent << bits_dccl_head_str << std::setfill('.')
600  << std::setw(bits_width - bits_dccl_head_str.size()) << id_bit_size << "\n"
601  << indent << bits_user_head_str << std::setfill('.')
602  << std::setw(bits_width - bits_user_head_str.size()) << config_head_bit_size << "\n"
603  << indent << bits_body_str << std::setfill('.')
604  << std::setw(bits_width - bits_body_str.size()) << body_bit_size << "\n"
605  << indent << bits_padding_str << std::setfill('.')
606  << std::setw(bits_width - bits_padding_str.size())
607  << byte_size * BITS_IN_BYTE - bit_size << "\n"
608  << "Allowed maximum size of message: " << allowed_byte_size << " bytes / "
609  << allowed_bit_size << " bits\n";
610 
611  std::string header_str = "Header";
612  std::string header_guard = build_guard_for_console_output(header_str, '-');
613 
614  *os << header_guard << " " << header_str << " " << header_guard << std::endl;
615  *os << bits_dccl_head_str << std::setfill('.')
616  << std::setw(bits_width - bits_dccl_head_str.size() + spaces) << id_bit_size << " {"
617  << id_codec()->name() << "}\n";
618  codec->base_info(os, desc, HEAD);
619  // *os << std::string(header_str.size() + 2 + 2*header_guard.size(), '-') << std::endl;
620 
621  std::string body_str = "Body";
622  std::string body_guard = build_guard_for_console_output(body_str, '-');
623 
624  *os << body_guard << " " << body_str << " " << body_guard << std::endl;
625  codec->base_info(os, desc, BODY);
626  // *os << std::string(body_str.size() + 2 + 2*body_guard.size(), '-') << std::endl;
627 
628  // *os << std::string(desc->full_name().size() + 2 + 2*guard.size(), '=') << std::endl;
629  }
630  catch (Exception& e)
631  {
632  dlog.is(DEBUG1) &&
633  dlog << "Message " << desc->full_name()
634  << " cannot provide information due to invalid configuration. Reason: "
635  << e.what() << std::endl;
636  }
637 
638  os->flush();
639  }
640 }
641 
642 void dccl::Codec::encrypt(std::string* s, const std::string& nonce /* message head */)
643 {
644 #if DCCL_HAS_CRYPTOPP
645  using namespace CryptoPP;
646 
647  std::string iv;
648  SHA256 hash;
649  StringSource unused(nonce, true, new HashFilter(hash, new StringSink(iv)));
650 
651  CTR_Mode<AES>::Encryption encryptor;
652  encryptor.SetKeyWithIV((byte*)crypto_key_.c_str(), crypto_key_.size(), (byte*)iv.c_str());
653 
654  std::string cipher;
655  StreamTransformationFilter in(encryptor, new StringSink(cipher));
656  in.Put((byte*)s->c_str(), s->size());
657  in.MessageEnd();
658  *s = cipher;
659 #endif
660 }
661 
662 void dccl::Codec::decrypt(std::string* s, const std::string& nonce)
663 {
664 #if DCCL_HAS_CRYPTOPP
665  using namespace CryptoPP;
666 
667  std::string iv;
668  SHA256 hash;
669  StringSource unused(nonce, true, new HashFilter(hash, new StringSink(iv)));
670 
671  CTR_Mode<AES>::Decryption decryptor;
672  decryptor.SetKeyWithIV((byte*)crypto_key_.c_str(), crypto_key_.size(), (byte*)iv.c_str());
673 
674  std::string recovered;
675 
676  StreamTransformationFilter out(decryptor, new StringSink(recovered));
677  out.Put((byte*)s->c_str(), s->size());
678  out.MessageEnd();
679  *s = recovered;
680 #endif
681 }
682 
683 void dccl::Codec::load_library(const std::string& library_path)
684 {
685  void* handle = dlopen(library_path.c_str(), RTLD_LAZY);
686  if (handle)
687  dl_handles_.push_back(handle);
688  load_library(handle);
689 }
690 
691 void dccl::Codec::load_library(void* dl_handle)
692 {
693  if (!dl_handle)
694  throw(Exception("Null shared library handle passed to load_library"));
695 
696  // load any shared library codecs
697  void (*dccl_load_ptr)(dccl::Codec*);
698  dccl_load_ptr = (void (*)(dccl::Codec*))dlsym(dl_handle, "dccl3_load");
699  if (dccl_load_ptr)
700  (*dccl_load_ptr)(this);
701 }
702 
703 void dccl::Codec::unload_library(void* dl_handle)
704 {
705  if (!dl_handle)
706  throw(Exception("Null shared library handle passed to unload_library"));
707 
708  // unload any shared library codecs
709  void (*dccl_unload_ptr)(dccl::Codec*);
710  dccl_unload_ptr = (void (*)(dccl::Codec*))dlsym(dl_handle, "dccl3_unload");
711  if (dccl_unload_ptr)
712  (*dccl_unload_ptr)(this);
713 }
714 
715 void dccl::Codec::set_crypto_passphrase(
716  const std::string& passphrase,
717  const std::set<unsigned>& do_not_encrypt_ids_ /*= std::set<unsigned>()*/)
718 {
719  if (!crypto_key_.empty())
720  crypto_key_.clear();
721  skip_crypto_ids_.clear();
722 
723 #if DCCL_HAS_CRYPTOPP
724  using namespace CryptoPP;
725 
726  SHA256 hash;
727  StringSource unused(passphrase, true, new HashFilter(hash, new StringSink(crypto_key_)));
728 
729  dlog.is(DEBUG1) && dlog << "Cryptography enabled with given passphrase" << std::endl;
730 #else
731  dlog.is(DEBUG1) && dlog << "Cryptography disabled because DCCL was compiled without support of "
732  "Crypto++. Install Crypto++ and recompile to enable cryptography."
733  << std::endl;
734 #endif
735 
736  skip_crypto_ids_ = do_not_encrypt_ids_;
737 }
738 
739 void dccl::Codec::info_all(std::ostream* param_os /*= 0 */) const
740 {
741  std::ostream* os = (param_os) ? param_os : &dlog;
742 
743  if (param_os || dlog.is(INFO))
744  {
745  std::string codec_str = "Dynamic Compact Control Language (DCCL) Codec";
746  std::string codec_guard = build_guard_for_console_output(codec_str, '|');
747 
748  *os << codec_guard << " " << codec_str << " " << codec_guard << std::endl;
749  *os << id2desc_.size() << " messages loaded.\n";
750  *os << "Field sizes are in bits unless otherwise noted." << std::endl;
751 
752  for (std::map<int32, const google::protobuf::Descriptor*>::const_iterator
753  it = id2desc_.begin(),
754  n = id2desc_.end();
755  it != n; ++it)
756  info(it->second, os, it->first);
757 
758  // *os << std::string(codec_str.size() + 2 + 2*codec_guard.size(), '|') << std::endl;
759  }
760 }
761 
762 void dccl::Codec::set_id_codec(const std::string& id_codec_name)
763 {
764  // we must reload messages after setting the id_codec
765  unload_all();
766 
767  id_codec_ = id_codec_name;
768  // make sure the id codec exists
769  id_codec();
770 }
771 
772 std::string dccl::Codec::build_guard_for_console_output(std::string& base, char guard_char) const
773 {
774  // Only guard if possible, otherwise return an empty string rather than throwing a std::length_error.
775  return (base.size() < console_width_)
776  ? std::string((console_width_ - base.size()) / 2, guard_char)
777  : std::string();
778 }
779 
780 std::string dccl::Codec::get_all_error_fields_in_message(const google::protobuf::Message& message,
781  uint8_t depth /*= 1 */)
782 {
783  // This is largely taken from google::protobuf::ReflectionOps::FindInitializationErrors(), which is called by
784  // google::protobuf::Message::FindInitializationErrors(). The Message implementation is not used as they force a
785  // prefix of parent message onto fields, which would require a split function to break by delimiter '.' should we
786  // want to reflect upon sub-messages and get field numbers.
787 
788  std::stringstream output_stream;
789 
790  const google::protobuf::Descriptor* descriptor = message.GetDescriptor();
791  const google::protobuf::Reflection* reflection = message.GetReflection();
792  const uint8_t depth_spacing = 4;
793 
794  // Check required fields of this message.
795  const int32_t field_count = descriptor->field_count();
796 
797  for (int32_t index = 0; index < field_count; ++index)
798  {
799  if (descriptor->field(index)->is_required())
800  {
801  if (!reflection->HasField(message, descriptor->field(index)))
802  {
803  output_stream << std::string(depth * depth_spacing, ' ')
804  << descriptor->field(index)->number() << ": "
805  << descriptor->field(index)->name() << "\n";
806  }
807  }
808  }
809 
810  // Check sub-messages.
811  std::vector<const google::protobuf::FieldDescriptor*> fields;
812  reflection->ListFields(message, &fields);
813 
814  for (const google::protobuf::FieldDescriptor* field : fields)
815  {
816  if (field->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE)
817  {
818  output_stream << std::string(depth * depth_spacing, ' ') << field->number() << ": "
819  << field->name() << "\n";
820 
821  if (field->is_repeated())
822  {
823  int32_t size = reflection->FieldSize(message, field);
824 
825  for (int32_t index = 0; index < size; ++index)
826  {
827  const google::protobuf::Message& sub_message =
828  reflection->GetRepeatedMessage(message, field, index);
829 
830  output_stream << std::string((depth + 1) * depth_spacing, ' ') << "[" << index
831  << "]: "
832  << "\n";
833 
834  output_stream << get_all_error_fields_in_message(sub_message, depth + 2);
835  }
836  }
837  else
838  {
839  const google::protobuf::Message& sub_message =
840  reflection->GetMessage(message, field);
841  output_stream << get_all_error_fields_in_message(sub_message, depth + 1);
842  }
843  }
844  }
845 
846  return output_stream.str();
847 }
dccl
Dynamic Compact Control Language namespace.
Definition: gen_units_class_plugin.h:49
dccl::Logger::is
bool is(logger::Verbosity verbosity, logger::Group group=logger::GENERAL)
Indicates the verbosity of the Logger until the next std::flush or std::endl. The boolean return is u...
Definition: logger.h:145
dccl::Exception
Exception class for DCCL.
Definition: exception.h:31
dccl::Codec::unload
void unload()
Unload a given message.
Definition: codec.h:103
dccl::Codec::set_id_codec
void set_id_codec(const std::string &id_codec_name)
Set a different ID codec name (note that is calls unload_all() so all messages must be reloaded)
Definition: codec.cpp:762
dccl::Codec::min_size
unsigned min_size()
Provides the encoded minimum size (in bytes) of msg.
Definition: codec.h:331
dccl::Codec::max_size
unsigned max_size()
Provides the encoded maximum size (in bytes) of msg.
Definition: codec.h:320
dccl::internal::MessageStack
Definition: field_codec_message_stack.h:41
dccl::Codec
The Dynamic CCL enCODer/DECoder. This is the main class you will use to load, encode and decode DCCL ...
Definition: codec.h:56
dccl::Codec::size
unsigned size(const google::protobuf::Message &msg, int user_id=-1)
Provides the encoded size (in bytes) of msg. This is useful if you need to know the size of a message...
Definition: codec.cpp:497
dccl::Codec::unload_library
void unload_library(void *dl_handle)
Remove codecs and/or unload messages present in the given shared library handle.
Definition: codec.cpp:703
dccl::hex_encode
void hex_encode(CharIterator begin, CharIterator end, std::string *out, bool upper_case=false)
Encodes a (little-endian) hexadecimal string from a byte string. Index 0 of begin is written to index...
Definition: binary.h:100
dccl::Bitset
A variable size container of bits (subclassed from std::deque<bool>) with an optional hierarchy....
Definition: bitset.h:38
dccl::OutOfRangeException
Definition: exception.h:48
dccl::Codec::load_library
void load_library(void *dl_handle)
Add codecs and/or load messages present in the given shared library handle.
Definition: codec.cpp:691
dccl::FieldCodecManager::find
static boost::shared_ptr< FieldCodecBase > find(const google::protobuf::FieldDescriptor *field, bool has_codec_group, const std::string &codec_group)
Find the codec for a given field. For embedded messages, prefers (dccl.field).codec (inside field) ov...
Definition: field_codec_manager.h:121
dccl::Codec::encode
void encode(std::string *bytes, const google::protobuf::Message &msg, bool header_only=false, int user_id=-1)
Encodes a DCCL message.
Definition: codec.cpp:331
dccl::Codec::set_crypto_passphrase
void set_crypto_passphrase(const std::string &passphrase, const std::set< unsigned > &do_not_encrypt_ids_=std::set< unsigned >())
Set a passphrase to be used when encoded messages to encrypt them and to decrypt messages after decod...
Definition: codec.cpp:715
Message
dccl::Codec::info
void info(std::ostream *os=0, int user_id=-1) const
Writes a human readable summary (including field sizes) of the provided DCCL type to the stream provi...
Definition: codec.h:170
dccl::Codec::load
void load()
All messages must be explicited loaded and validated (size checks, option extensions checks,...
Definition: codec.h:96
dccl::hex_decode
void hex_decode(const std::string &in, std::string *out)
Decodes a (little-endian) hexadecimal string to a byte string. Index 0 and 1 (first byte) of in are w...
Definition: binary.h:49
dccl::Codec::Codec
Codec(const std::string &dccl_id_codec=default_id_codec_name(), const std::string &library_path="")
Instantiate a Codec, optionally with a non-default identifier field codec.
Definition: codec.cpp:66
dccl::Codec::decode
CharIterator decode(CharIterator begin, CharIterator end, google::protobuf::Message *msg, bool header_only=false)
Decode a DCCL message when the type is known at compile time.
Definition: codec.h:462
dccl::Codec::id
unsigned id() const
Gives the DCCL id (defined by the custom message option extension "(dccl.msg).id" in the ....
Definition: codec.h:189
dccl::Codec::info_all
void info_all(std::ostream *os=0) const
Writes a human readable summary (including field sizes) of all the loaded (validated) DCCL types.
Definition: codec.cpp:739
dccl::Codec::~Codec
virtual ~Codec()
Destructor.
Definition: codec.cpp:78
dccl::Bitset::to_byte_string
std::string to_byte_string()
Returns the value of the Bitset to a byte string, where each character represents 8 bits of the Bitse...
Definition: bitset.h:310