codec.h

// Copyright 2009-2023:
//   GobySoft, LLC (2013-)
//   Massachusetts Institute of Technology (2007-2014)
//   Community contributors (see AUTHORS file)
// File authors:
//   Toby Schneider <toby@gobysoft.org>
//   Nathan Knotts <nknotts@gmail.com>
//   philboske <philboske@gmail.com>
//   Chris Murphy <cmurphy@aphysci.com>
//
//
// This file is part of the Dynamic Compact Control Language Library
// ("DCCL").
//
// DCCL is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// DCCL is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with DCCL.  If not, see <http://www.gnu.org/licenses/>.
#ifndef DCCL20091211H
#define DCCL20091211H
 
#include <map>
#include <ostream>
#include <set>
#include <stdexcept>
#include <string>
#include <type_traits>
#include <vector>
 
#include <google/protobuf/descriptor.h>
 
#include <memory>
 
#include "binary.h"
#include "dynamic_protobuf_manager.h"
#include "exception.h"
#include "field_codec.h"
#include "field_codec_fixed.h"
#include "logger.h"
 
#include "dccl/def.h"
#include "dccl/version.h"
#include "field_codec_manager.h"
 
namespace dccl
{
class FieldCodec;
 
class Codec
{
  public:
    Codec(std::string dccl_id_codec_name = default_id_codec_name(),
          const std::string& library_path = "");
 
    template <class IDFieldCodec,
              typename std::enable_if<std::is_base_of<FieldCodecBase, IDFieldCodec>::value,
                                      int>::type = 0>
    Codec(const std::string& dccl_id_codec_name, const IDFieldCodec& dccl_id_codec) // NOLINT
        : id_codec_(dccl_id_codec_name)
    {
        set_default_codecs();
        manager_.add<IDFieldCodec>(dccl_id_codec_name);
    }
 
    virtual ~Codec();
 
    Codec(const Codec&) = delete;
    Codec& operator=(const Codec&) = delete;
 
    void load_library(void* dl_handle);
 
    void unload_library(void* dl_handle);
 
    void load_library(const std::string& library_path);
 
    template <typename ProtobufMessage> std::size_t load()
    {
        return load(ProtobufMessage::descriptor());
    }
 
    template <typename ProtobufMessage> void unload() { unload(ProtobufMessage::descriptor()); }
 
    void unload_all() { id2desc_.clear(); }
 
    std::size_t load(const google::protobuf::Descriptor* desc, int user_id = -1);
 
    void unload(const google::protobuf::Descriptor* desc);
 
    void unload(size_t dccl_id);
 
    void set_id_codec(const std::string& id_codec_name);
    std::string get_id_codec() { return id_codec_; }
 
    void set_crypto_passphrase(const std::string& passphrase,
                               const std::set<int32>& do_not_encrypt_ids = std::set<int32>());
 
    void set_crypto_passphrase(const std::string& passphrase,
                               const std::set<unsigned>& do_not_encrypt_ids)
    {
        std::set<int32> s_ids{do_not_encrypt_ids.begin(), do_not_encrypt_ids.end()};
        set_crypto_passphrase(passphrase, s_ids);
    }
 
    void set_strict(bool mode) { strict_ = mode; }
 
    void set_console_width(unsigned num_chars) { console_width_ = num_chars; }
 
 
 
 
    template <typename ProtobufMessage>
    void info(std::ostream* os = nullptr, int user_id = -1) const
    {
        info(ProtobufMessage::descriptor(), os, user_id);
    }
 
    void info(const google::protobuf::Descriptor* desc, std::ostream* os = nullptr,
              int user_id = -1) const;
 
    void info_all(std::ostream* os = nullptr) const;
 
    template <typename ProtobufMessage> int32 id() const
    {
        return id(ProtobufMessage::descriptor());
    }
 
    int32 id(const std::string& bytes) const;
 
    template <typename CharIterator> int32 id(CharIterator begin, CharIterator end) const;
 
    int32 id(const google::protobuf::Descriptor* desc) const
    {
        if (desc->options().GetExtension(dccl::msg).omit_id())
            throw(Exception("Cannot call id(...) on message with omit_id == true"));
        Bitset id_bits;
        dccl::uint32 hardcoded_id = desc->options().GetExtension(dccl::msg).id();
        // pass the hard coded id, that is, (dccl.msg).id,
        // through encode/decode to allow a custom ID codec (if in use)
        // to always take effect.
        id_codec()->field_encode(&id_bits, hardcoded_id, nullptr);
        std::string id_bytes(id_bits.to_byte_string());
        return id(id_bytes);
    }
 
    const std::map<int32, const google::protobuf::Descriptor*>& loaded() const { return id2desc_; }
 
 
 
 
    void encode(std::string* bytes, const google::protobuf::Message& msg, bool header_only = false,
                int user_id = -1);
 
    size_t encode(char* bytes, size_t max_len, const google::protobuf::Message& msg,
                  bool header_only = false, int user_id = -1);
 
    template <typename CharIterator, typename ProtobufMessage>
    CharIterator decode(CharIterator begin, CharIterator end, ProtobufMessage* msg,
                        bool header_only = false);
 
    template <typename ProtobufMessage>
    void decode(const std::string& bytes, ProtobufMessage* msg, bool header_only = false)
    {
        decode(bytes.begin(), bytes.end(), msg, header_only);
    }
 
    template <typename ProtobufMessage> void decode(std::string* bytes, ProtobufMessage* msg)
    {
        decode(*bytes, msg);
        unsigned last_size = size(*msg);
        bytes->erase(0, last_size);
    }
 
    template <typename GoogleProtobufMessagePointer>
    GoogleProtobufMessagePointer decode(const std::string& bytes, bool header_only = false);
 
    template <typename GoogleProtobufMessagePointer>
    GoogleProtobufMessagePointer decode(std::string* bytes);
 
    unsigned size(const google::protobuf::Message& msg, int user_id = -1);
 
    template <typename ProtobufMessage> unsigned max_size()
    {
        return max_size(ProtobufMessage::descriptor());
    }
 
    unsigned max_size(const google::protobuf::Descriptor* desc, int user_id = -1) const;
 
    template <typename ProtobufMessage> unsigned min_size()
    {
        return min_size(ProtobufMessage::descriptor());
    }
 
    unsigned min_size(const google::protobuf::Descriptor* desc, int user_id = -1) const;
 
 
    static std::string default_id_codec_name() { return "dccl.default.id"; }
 
    static std::string default_codec_name(int version = 2)
    {
        return "dccl.default" + std::to_string(version);
    }
 
    FieldCodecManagerLocal& manager() { return manager_; }
 
  private:
    void encode_internal(const google::protobuf::Message& msg, bool header_only,
                         Bitset& header_bits, Bitset& body_bits, int user_id);
    std::string get_all_error_fields_in_message(const google::protobuf::Message& msg,
                                                uint8_t depth = 1);
 
    void encrypt(std::string* s, const std::string& nonce);
    void decrypt(std::string* s, const std::string& nonce);
 
    void set_default_codecs();
 
    std::shared_ptr<FieldCodecBase> id_codec() const
    {
        return manager_.find(google::protobuf::FieldDescriptor::TYPE_UINT32, DCCL_VERSION_MAJOR,
                             id_codec_);
    }
 
    int32 id_internal(const google::protobuf::Descriptor* desc, int user_id)
    {
        // if we have omit_id, check for or assign an autogenerate negative internal placeholder ID
        if (desc->options().GetExtension(dccl::msg).omit_id() && !desc2placeholder_id_.count(desc))
            desc2placeholder_id_.emplace(desc, omit_id_placeholder_id_--);
 
        return id_internal_const(desc, user_id);
    }
 
    int32 id_internal_const(const google::protobuf::Descriptor* desc, int user_id) const
    {
        if (desc->options().GetExtension(dccl::msg).omit_id())
        {
            if (desc2placeholder_id_.count(desc))
                return desc2placeholder_id_.find(desc)->second;
            else
                throw(Exception("Message " + std::string(desc->full_name()) +
                                " has omit_id == true but has not been loaded, so id_internal() "
                                "const cannot be called"));
        }
        else
        {
            return (user_id < 0) ? id(desc) : user_id;
        }
    }
 
  private:
    // SHA256 hash of the crypto passphrase
    std::string crypto_key_;
 
    // strict mode setting
    bool strict_{false};
 
    // console outputting format width
    unsigned console_width_{60};
 
    // set of DCCL IDs *not* to encrypt
    std::set<int32> skip_crypto_ids_;
 
    // maps `dccl.id`s onto Message Descriptors
    std::map<int32, const google::protobuf::Descriptor*> id2desc_;
    std::string id_codec_;
 
    std::vector<void*> dl_handles_;
 
    std::string build_guard_for_console_output(std::string& base, char guard_char) const;
 
    FieldCodecManagerLocal manager_;
 
    // current omit_id placeholder DCCL Id (starts at -1 and decrements)
    int32 omit_id_placeholder_id_{-1};
    // maps message descriptor onto placeholder ID for omit_id messages
    std::map<const google::protobuf::Descriptor*, int32> desc2placeholder_id_;
};
 
inline std::ostream& operator<<(std::ostream& os, const Codec& codec)
{
    codec.info_all(&os);
    return os;
}
} // namespace dccl
 
template <typename GoogleProtobufMessagePointer>
GoogleProtobufMessagePointer dccl::Codec::decode(const std::string& bytes,
                                                 bool header_only /* = false */)
{
    int32 this_id = id(bytes);
 
    if (!id2desc_.count(this_id))
        throw(Exception("Message id " + std::to_string(this_id) +
                        " has not been loaded. Call load() before decoding this type."));
 
    // ownership of this object goes to the caller of decode()
    auto msg = dccl::DynamicProtobufManager::new_protobuf_message<GoogleProtobufMessagePointer>(
        id2desc_.find(this_id)->second);
    decode(bytes, &(*msg), header_only);
    return msg;
}
 
template <typename GoogleProtobufMessagePointer>
GoogleProtobufMessagePointer dccl::Codec::decode(std::string* bytes)
{
    int32 this_id = id(*bytes);
 
    if (!id2desc_.count(this_id))
        throw(Exception("Message id " + std::to_string(this_id) +
                        " has not been loaded. Call load() before decoding this type."));
 
    GoogleProtobufMessagePointer msg =
        dccl::DynamicProtobufManager::new_protobuf_message<GoogleProtobufMessagePointer>(
            id2desc_.find(this_id)->second);
    std::string::iterator new_begin = decode(bytes->begin(), bytes->end(), &(*msg));
    bytes->erase(bytes->begin(), new_begin);
    return msg;
}
 
template <typename CharIterator>
dccl::int32 dccl::Codec::id(CharIterator begin, CharIterator end) const
{
    try
    {
        unsigned id_min_size = 0, id_max_size = 0;
        id_codec()->field_min_size(&id_min_size, nullptr);
        id_codec()->field_max_size(&id_max_size, nullptr);
        Bitset fixed_header_bits;
 
        // ensure we don't go past-the-end if fewer bytes are passed in than id_max_size
        int incr = std::min<size_t>(
            static_cast<size_t>(std::distance(begin, end)),
            static_cast<size_t>(std::ceil(static_cast<double>(id_max_size) / BITS_IN_BYTE)));
        fixed_header_bits.from_byte_stream(begin, begin + incr);
 
        Bitset these_bits(&fixed_header_bits);
        these_bits.get_more_bits(id_min_size);
 
        dccl::any return_value;
        id_codec()->field_decode(&these_bits, &return_value, nullptr);
        return dccl::any_cast<uint32>(return_value);
    }
    catch (const dccl::Exception& e)
    {
        throw(Exception("Failed to decoded id from bytes passed (hex: " + hex_encode(begin, end) +
                        ")"));
    }
}
 
template <typename CharIterator, typename ProtobufMessage>
CharIterator dccl::Codec::decode(CharIterator begin, CharIterator end, ProtobufMessage* msg,
                                 bool header_only /*= false*/)
{
    try
    {
        const google::protobuf::Descriptor* desc = msg->GetDescriptor();
        int32 expected_id = id_internal(desc, -1);
        int32 received_id =
            expected_id; // if omit_id, we have to assume we have the correct type. Otherwise, overwrite if not omit_id and check
        if (!desc->options().GetExtension(dccl::msg).omit_id())
        {
            received_id = id(begin, end);
 
            if (!id2desc_.count(received_id))
                throw(Exception("Message id " + std::to_string(received_id) +
                                " has not been loaded. Call load() before decoding this type."));
 
            if (expected_id != received_id)
                throw(Exception("Received message with id " + std::to_string(received_id) + " (" +
                                std::string(id2desc_.at(received_id)->full_name()) +
                                ") but decode was called with message of id " +
                                std::to_string(expected_id) + " (" +
                                std::string(desc->full_name()) +
                                "). Ensure dccl::Codec::decode is called with the correct Protobuf "
                                "message or use the dynamic overloads of decode."));
        }
 
        dlog.is(logger::DEBUG1, logger::DECODE) &&
            dlog << "Began decoding message of id: " << received_id << std::endl;
 
        dlog.is(logger::DEBUG1, logger::DECODE) && dlog << "Type name: " << desc->full_name()
                                                        << std::endl;
 
        std::shared_ptr<FieldCodecBase> codec = manager_.find(desc);
        std::shared_ptr<internal::FromProtoCppTypeBase> helper = manager_.type_helper().find(desc);
 
        CharIterator actual_end = end;
        if (codec)
        {
            unsigned head_size_bits;
            unsigned body_size_bits;
            codec->base_max_size(&head_size_bits, desc, HEAD);
            codec->base_max_size(&body_size_bits, desc, BODY);
            unsigned id_size = 0;
            if (!desc->options().GetExtension(dccl::msg).omit_id())
                id_codec()->field_size(&id_size, static_cast<uint32>(received_id), nullptr);
            head_size_bits += id_size;
 
            unsigned head_size_bytes = ceil_bits2bytes(head_size_bits);
            unsigned body_size_bytes = ceil_bits2bytes(body_size_bits);
 
            dlog.is(logger::DEBUG2, logger::DECODE) &&
                dlog << "Head bytes (bits): " << head_size_bytes << "(" << head_size_bits
                     << "), max body bytes (bits): " << body_size_bytes << "(" << body_size_bits
                     << ")" << std::endl;
 
            CharIterator head_bytes_end = begin + head_size_bytes;
            dlog.is(logger::DEBUG3, logger::DECODE) &&
                dlog << "Unencrypted Head (hex): " << hex_encode(begin, head_bytes_end)
                     << std::endl;
 
            Bitset head_bits;
            head_bits.from_byte_stream(begin, head_bytes_end);
            dlog.is(logger::DEBUG3, logger::DECODE) &&
                dlog << "Unencrypted Head (bin): " << head_bits << std::endl;
 
            // shift off ID bits
            head_bits >>= id_size;
 
            dlog.is(logger::DEBUG3, logger::DECODE) &&
                dlog << "Unencrypted Head after ID bits removal (bin): " << head_bits << std::endl;
 
            internal::MessageStack msg_stack(manager_.codec_data().root_message_,
                                             manager_.codec_data().message_data_);
            msg_stack.push(msg->GetDescriptor());
 
            codec->base_decode(&head_bits, msg, HEAD);
            dlog.is(logger::DEBUG2, logger::DECODE) &&
                dlog << "after header decode, message is: " << *msg << std::endl;
 
            if (header_only)
            {
                dlog.is(logger::DEBUG2, logger::DECODE) &&
                    dlog << "as requested, skipping decrypting and decoding body." << std::endl;
                actual_end = head_bytes_end;
            }
            else
            {
                dlog.is(logger::DEBUG3, logger::DECODE) &&
                    dlog << "Encrypted Body (hex): " << hex_encode(head_bytes_end, end)
                         << std::endl;
 
                Bitset body_bits;
                if (!crypto_key_.empty() && !skip_crypto_ids_.count(received_id))
                {
                    std::string head_bytes(begin, head_bytes_end);
                    std::string body_bytes(head_bytes_end, end);
                    decrypt(&body_bytes, head_bytes);
                    dlog.is(logger::DEBUG3, logger::DECODE) &&
                        dlog << "Unencrypted Body (hex): " << hex_encode(body_bytes) << std::endl;
                    body_bits.from_byte_stream(body_bytes.begin(), body_bytes.end());
                }
                else
                {
                    dlog.is(logger::DEBUG3, logger::DECODE) &&
                        dlog << "Unencrypted Body (hex): " << hex_encode(head_bytes_end, end)
                             << std::endl;
                    body_bits.from_byte_stream(head_bytes_end, end);
                }
 
                dlog.is(logger::DEBUG3, logger::DECODE) &&
                    dlog << "Unencrypted Body (bin): " << body_bits << std::endl;
 
                codec->base_decode(&body_bits, msg, BODY);
                dlog.is(logger::DEBUG2, logger::DECODE) &&
                    dlog << "after header & body decode, message is: " << *msg << std::endl;
 
                actual_end = end - body_bits.size() / BITS_IN_BYTE;
            }
        }
        else
        {
            throw(Exception("Failed to find (dccl.msg).codec `" +
                            desc->options().GetExtension(dccl::msg).codec() + "`"),
                  desc);
        }
 
        dlog.is(logger::DEBUG1, logger::DECODE) &&
            dlog << "Successfully decoded message of type: " << desc->full_name() << std::endl;
        return actual_end;
    }
    catch (std::exception& e)
    {
        std::stringstream ss;
 
        ss << "Message " << hex_encode(begin, end) << " failed to decode. Reason: " << e.what()
           << std::endl;
 
        dlog.is(logger::DEBUG1, logger::DECODE) && dlog << ss.str() << std::endl;
        throw(Exception(ss.str()));
    }
}
 
#endif