codec.h

// Copyright 2009-2023:
//   GobySoft, LLC (2013-)
//   Massachusetts Institute of Technology (2007-2014)
//   Community contributors (see AUTHORS file)
// File authors:
//   Toby Schneider <toby@gobysoft.org>
//   Nathan Knotts <nknotts@gmail.com>
//   philboske <philboske@gmail.com>
//   Chris Murphy <cmurphy@aphysci.com>
//
//
// This file is part of the Dynamic Compact Control Language Library
// ("DCCL").
//
// DCCL is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// DCCL is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with DCCL.  If not, see <http://www.gnu.org/licenses/>.
#ifndef DCCL20091211H
#define DCCL20091211H
 
#include <map>
#include <ostream>
#include <set>
#include <stdexcept>
#include <string>
#include <type_traits>
#include <vector>
 
#include <google/protobuf/descriptor.h>
 
#include <memory>
 
#include "binary.h"
#include "dynamic_protobuf_manager.h"
#include "exception.h"
#include "field_codec.h"
#include "field_codec_fixed.h"
#include "logger.h"
 
#include "codecs2/field_codec_default_message.h"
#include "codecs3/field_codec_default_message.h"
#include "dccl/def.h"
#include "field_codec_manager.h"
 
namespace dccl
{
class FieldCodec;
 
class Codec
{
  public:
    Codec(std::string dccl_id_codec_name = default_id_codec_name(),
          const std::string& library_path = "");
 
    template <class IDFieldCodec,
              typename std::enable_if<std::is_base_of<FieldCodecBase, IDFieldCodec>::value,
                                      int>::type = 0>
    Codec(const std::string& dccl_id_codec_name, const IDFieldCodec& dccl_id_codec) // NOLINT
        : id_codec_(dccl_id_codec_name)
    {
        set_default_codecs();
        manager_.add<IDFieldCodec>(dccl_id_codec_name);
    }
 
    virtual ~Codec();
 
    Codec(const Codec&) = delete;
    Codec& operator=(const Codec&) = delete;
 
    void load_library(void* dl_handle);
 
    void unload_library(void* dl_handle);
 
    void load_library(const std::string& library_path);
 
    template <typename ProtobufMessage> void load() { load(ProtobufMessage::descriptor()); }
 
    template <typename ProtobufMessage> void unload() { unload(ProtobufMessage::descriptor()); }
 
    void unload_all() { id2desc_.clear(); }
 
    void load(const google::protobuf::Descriptor* desc, int user_id = -1);
 
    void unload(const google::protobuf::Descriptor* desc);
 
    void unload(size_t dccl_id);
 
    void set_id_codec(const std::string& id_codec_name);
    std::string get_id_codec() { return id_codec_; }
 
    void
    set_crypto_passphrase(const std::string& passphrase,
                          const std::set<unsigned>& do_not_encrypt_ids_ = std::set<unsigned>());
 
    void set_strict(bool mode) { strict_ = mode; }
 
    void set_console_width(unsigned num_chars) { console_width_ = num_chars; }
 
 
 
 
    template <typename ProtobufMessage>
    void info(std::ostream* os = nullptr, int user_id = -1) const
    {
        info(ProtobufMessage::descriptor(), os, user_id);
    }
 
    void info(const google::protobuf::Descriptor* desc, std::ostream* os = nullptr,
              int user_id = -1) const;
 
    void info_all(std::ostream* os = nullptr) const;
 
    template <typename ProtobufMessage> unsigned id() const
    {
        return id(ProtobufMessage::descriptor());
    }
 
    unsigned id(const std::string& bytes) const;
 
    template <typename CharIterator> unsigned id(CharIterator begin, CharIterator end) const;
 
    unsigned id(const google::protobuf::Descriptor* desc) const
    {
        Bitset id_bits;
        dccl::uint32 hardcoded_id = desc->options().GetExtension(dccl::msg).id();
        // pass the hard coded id, that is, (dccl.msg).id,
        // through encode/decode to allow a custom ID codec (if in use)
        // to always take effect.
        id_codec()->field_encode(&id_bits, hardcoded_id, nullptr);
        std::string id_bytes(id_bits.to_byte_string());
        return id(id_bytes);
    }
 
    const std::map<int32, const google::protobuf::Descriptor*>& loaded() const { return id2desc_; }
 
 
 
 
    void encode(std::string* bytes, const google::protobuf::Message& msg, bool header_only = false,
                int user_id = -1);
 
    size_t encode(char* bytes, size_t max_len, const google::protobuf::Message& msg,
                  bool header_only = false, int user_id = -1);
 
    template <typename CharIterator>
    CharIterator decode(CharIterator begin, CharIterator end, google::protobuf::Message* msg,
                        bool header_only = false);
 
    void decode(const std::string& bytes, google::protobuf::Message* msg, bool header_only = false);
 
    void decode(std::string* bytes, google::protobuf::Message* msg);
 
    template <typename GoogleProtobufMessagePointer>
    GoogleProtobufMessagePointer decode(const std::string& bytes, bool header_only = false);
 
    template <typename GoogleProtobufMessagePointer>
    GoogleProtobufMessagePointer decode(std::string* bytes);
 
    unsigned size(const google::protobuf::Message& msg, int user_id = -1);
 
    template <typename ProtobufMessage> unsigned max_size()
    {
        return max_size(ProtobufMessage::descriptor());
    }
 
    unsigned max_size(const google::protobuf::Descriptor* desc) const;
 
    template <typename ProtobufMessage> unsigned min_size()
    {
        return min_size(ProtobufMessage::descriptor());
    }
 
    unsigned min_size(const google::protobuf::Descriptor* desc) const;
 
 
    static std::string default_id_codec_name() { return "dccl.default.id"; }
 
    static std::string default_codec_name(int version = 2)
    {
        switch (version)
        {
            case 2:
                return dccl::DCCLFieldOptions::descriptor()
                    ->FindFieldByName("codec")
                    ->default_value_string();
            default: return "dccl.default" + std::to_string(version);
        }
    }
 
    FieldCodecManagerLocal& manager() { return manager_; }
 
  private:
    void encode_internal(const google::protobuf::Message& msg, bool header_only,
                         Bitset& header_bits, Bitset& body_bits, int user_id);
    std::string get_all_error_fields_in_message(const google::protobuf::Message& msg,
                                                uint8_t depth = 1);
 
    void encrypt(std::string* s, const std::string& nonce);
    void decrypt(std::string* s, const std::string& nonce);
 
    void set_default_codecs();
 
    std::shared_ptr<FieldCodecBase> id_codec() const
    {
        return manager_.find(google::protobuf::FieldDescriptor::TYPE_UINT32, id_codec_);
    }
 
  private:
    // SHA256 hash of the crypto passphrase
    std::string crypto_key_;
 
    // strict mode setting
    bool strict_{false};
 
    // console outputting format width
    unsigned console_width_{60};
 
    // set of DCCL IDs *not* to encrypt
    std::set<unsigned> skip_crypto_ids_;
 
    // maps `dccl.id`s onto Message Descriptors
    std::map<int32, const google::protobuf::Descriptor*> id2desc_;
    std::string id_codec_;
 
    std::vector<void*> dl_handles_;
 
    std::string build_guard_for_console_output(std::string& base, char guard_char) const;
 
    FieldCodecManagerLocal manager_;
};
 
inline std::ostream& operator<<(std::ostream& os, const Codec& codec)
{
    codec.info_all(&os);
    return os;
}
} // namespace dccl
 
template <typename GoogleProtobufMessagePointer>
GoogleProtobufMessagePointer dccl::Codec::decode(const std::string& bytes,
                                                 bool header_only /* = false */)
{
    unsigned this_id = id(bytes);
 
    if (!id2desc_.count(this_id))
        throw(Exception("Message id " + std::to_string(this_id) +
                        " has not been loaded. Call load() before decoding this type."));
 
    // ownership of this object goes to the caller of decode()
    auto msg = dccl::DynamicProtobufManager::new_protobuf_message<GoogleProtobufMessagePointer>(
        id2desc_.find(this_id)->second);
    decode(bytes, &(*msg), header_only);
    return msg;
}
 
template <typename GoogleProtobufMessagePointer>
GoogleProtobufMessagePointer dccl::Codec::decode(std::string* bytes)
{
    unsigned this_id = id(*bytes);
 
    if (!id2desc_.count(this_id))
        throw(Exception("Message id " + std::to_string(this_id) +
                        " has not been loaded. Call load() before decoding this type."));
 
    GoogleProtobufMessagePointer msg =
        dccl::DynamicProtobufManager::new_protobuf_message<GoogleProtobufMessagePointer>(
            id2desc_.find(this_id)->second);
    std::string::iterator new_begin = decode(bytes->begin(), bytes->end(), &(*msg));
    bytes->erase(bytes->begin(), new_begin);
    return msg;
}
 
template <typename CharIterator>
unsigned dccl::Codec::id(CharIterator begin, CharIterator end) const
{
    unsigned id_min_size = 0, id_max_size = 0;
    id_codec()->field_min_size(&id_min_size, nullptr);
    id_codec()->field_max_size(&id_max_size, nullptr);
 
    if (std::distance(begin, end) < (id_min_size / BITS_IN_BYTE))
        throw(Exception("Bytes passed (hex: " + hex_encode(begin, end) +
                        ") is too small to be a valid DCCL message"));
 
    Bitset fixed_header_bits;
    fixed_header_bits.from_byte_stream(
        begin, begin + (size_t)std::ceil(double(id_max_size) / BITS_IN_BYTE));
 
    Bitset these_bits(&fixed_header_bits);
    these_bits.get_more_bits(id_min_size);
 
    dccl::any return_value;
    id_codec()->field_decode(&these_bits, &return_value, nullptr);
 
    return dccl::any_cast<uint32>(return_value);
}
 
template <typename CharIterator>
CharIterator dccl::Codec::decode(CharIterator begin, CharIterator end,
                                 google::protobuf::Message* msg, bool header_only /*= false*/)
{
    try
    {
        unsigned this_id = id(begin, end);
 
        dlog.is(logger::DEBUG1, logger::DECODE) &&
            dlog << "Began decoding message of id: " << this_id << std::endl;
 
        if (!id2desc_.count(this_id))
            throw(Exception("Message id " + std::to_string(this_id) +
                            " has not been loaded. Call load() before decoding this type."));
 
        const google::protobuf::Descriptor* desc = msg->GetDescriptor();
 
        dlog.is(logger::DEBUG1, logger::DECODE) && dlog << "Type name: " << desc->full_name()
                                                        << std::endl;
 
        std::shared_ptr<FieldCodecBase> codec = manager_.find(desc);
        std::shared_ptr<internal::FromProtoCppTypeBase> helper = manager_.type_helper().find(desc);
 
        CharIterator actual_end = end;
        if (codec)
        {
            unsigned head_size_bits;
            unsigned body_size_bits;
            codec->base_max_size(&head_size_bits, desc, HEAD);
            codec->base_max_size(&body_size_bits, desc, BODY);
            unsigned id_size = 0;
            id_codec()->field_size(&id_size, this_id, nullptr);
            head_size_bits += id_size;
 
            unsigned head_size_bytes = ceil_bits2bytes(head_size_bits);
            unsigned body_size_bytes = ceil_bits2bytes(body_size_bits);
 
            dlog.is(logger::DEBUG2, logger::DECODE) &&
                dlog << "Head bytes (bits): " << head_size_bytes << "(" << head_size_bits
                     << "), max body bytes (bits): " << body_size_bytes << "(" << body_size_bits
                     << ")" << std::endl;
 
            CharIterator head_bytes_end = begin + head_size_bytes;
            dlog.is(logger::DEBUG3, logger::DECODE) &&
                dlog << "Unencrypted Head (hex): " << hex_encode(begin, head_bytes_end)
                     << std::endl;
 
            Bitset head_bits;
            head_bits.from_byte_stream(begin, head_bytes_end);
            dlog.is(logger::DEBUG3, logger::DECODE) &&
                dlog << "Unencrypted Head (bin): " << head_bits << std::endl;
 
            // shift off ID bits
            head_bits >>= id_size;
 
            dlog.is(logger::DEBUG3, logger::DECODE) &&
                dlog << "Unencrypted Head after ID bits removal (bin): " << head_bits << std::endl;
 
            internal::MessageStack msg_stack(manager_.codec_data().root_message_,
                                             manager_.codec_data().message_data_);
            msg_stack.push(msg->GetDescriptor());
 
            codec->base_decode(&head_bits, msg, HEAD);
            dlog.is(logger::DEBUG2, logger::DECODE) &&
                dlog << "after header decode, message is: " << *msg << std::endl;
 
            if (header_only)
            {
                dlog.is(logger::DEBUG2, logger::DECODE) &&
                    dlog << "as requested, skipping decrypting and decoding body." << std::endl;
                actual_end = head_bytes_end;
            }
            else
            {
                dlog.is(logger::DEBUG3, logger::DECODE) &&
                    dlog << "Encrypted Body (hex): " << hex_encode(head_bytes_end, end)
                         << std::endl;
 
                Bitset body_bits;
                if (!crypto_key_.empty() && !skip_crypto_ids_.count(this_id))
                {
                    std::string head_bytes(begin, head_bytes_end);
                    std::string body_bytes(head_bytes_end, end);
                    decrypt(&body_bytes, head_bytes);
                    dlog.is(logger::DEBUG3, logger::DECODE) &&
                        dlog << "Unencrypted Body (hex): " << hex_encode(body_bytes) << std::endl;
                    body_bits.from_byte_stream(body_bytes.begin(), body_bytes.end());
                }
                else
                {
                    dlog.is(logger::DEBUG3, logger::DECODE) &&
                        dlog << "Unencrypted Body (hex): " << hex_encode(head_bytes_end, end)
                             << std::endl;
                    body_bits.from_byte_stream(head_bytes_end, end);
                }
 
                dlog.is(logger::DEBUG3, logger::DECODE) &&
                    dlog << "Unencrypted Body (bin): " << body_bits << std::endl;
 
                codec->base_decode(&body_bits, msg, BODY);
                dlog.is(logger::DEBUG2, logger::DECODE) &&
                    dlog << "after header & body decode, message is: " << *msg << std::endl;
 
                actual_end = end - body_bits.size() / BITS_IN_BYTE;
            }
        }
        else
        {
            throw(Exception("Failed to find (dccl.msg).codec `" +
                            desc->options().GetExtension(dccl::msg).codec() + "`"),
                  desc);
        }
 
        dlog.is(logger::DEBUG1, logger::DECODE) &&
            dlog << "Successfully decoded message of type: " << desc->full_name() << std::endl;
        return actual_end;
    }
    catch (std::exception& e)
    {
        std::stringstream ss;
 
        ss << "Message " << hex_encode(begin, end) << " failed to decode. Reason: " << e.what()
           << std::endl;
 
        dlog.is(logger::DEBUG1, logger::DECODE) && dlog << ss.str() << std::endl;
        throw(Exception(ss.str()));
    }
}
 
#endif