codec.h

// Copyright 2009-2017 Toby Schneider (http://gobysoft.org/index.wt/people/toby)
//                     GobySoft, LLC (for 2013-)
//                     Massachusetts Institute of Technology (for 2007-2014)
//                     Community contributors (see AUTHORS file)
//
//
// This file is part of the Dynamic Compact Control Language Library
// ("DCCL").
//
// DCCL is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// DCCL is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with DCCL.  If not, see <http://www.gnu.org/licenses/>.
#ifndef DCCL20091211H
#define DCCL20091211H
 
#include <string>
#include <set>
#include <map>
#include <ostream>
#include <stdexcept>
#include <vector>
 
#include <google/protobuf/descriptor.h>
 
#include <boost/shared_ptr.hpp>
 
#include "binary.h"
#include "dynamic_protobuf_manager.h"
#include "logger.h"
#include "exception.h"
#include "field_codec.h"
#include "field_codec_fixed.h"
 
#include "codecs2/field_codec_default_message.h"
#include "codecs3/field_codec_default_message.h"
#include "field_codec_manager.h"
 
#define DCCL_HAS_CRYPTOPP 1
 
namespace dccl
{
    class FieldCodec;
  
    class Codec
    {
      public:       
        Codec(const std::string& dccl_id_codec = default_id_codec_name(),
              const std::string& library_path = "");
 
        virtual ~Codec();
 
        void load_library(void* dl_handle);
 
        void unload_library(void* dl_handle);
 
        void load_library(const std::string& library_path);
        
        template<typename ProtobufMessage>
            void load()
        { load(ProtobufMessage::descriptor()); }
 
        template<typename ProtobufMessage>
            void unload()
        { unload(ProtobufMessage::descriptor()); }
 
 
        void unload_all()
        { id2desc_.clear(); }
        
        void load(const google::protobuf::Descriptor* desc, int user_id = - 1);
 
        void unload(const google::protobuf::Descriptor* desc);
 
        void unload(size_t dccl_id);
 
        void set_id_codec(const std::string& id_codec_name);
        std::string get_id_codec() { return id_codec_; }
 
        
        void set_crypto_passphrase(const std::string& passphrase,
                                   const std::set<unsigned>& do_not_encrypt_ids_ = std::set<unsigned>());
 
        
        void set_strict(bool mode) { strict_ = mode; }
        
            
 
 
        template<typename ProtobufMessage>
            void info(std::ostream* os = 0, int user_id = -1) const
        { info(ProtobufMessage::descriptor(), os, user_id); }
 
        void info(const google::protobuf::Descriptor* desc, std::ostream* os = 0, int user_id = - 1) const;
 
        void info_all(std::ostream* os = 0) const;
            
        template <typename ProtobufMessage>
            unsigned id() const
        { return id(ProtobufMessage::descriptor()); }            
 
        unsigned id(const std::string& bytes) const;
 
        template<typename CharIterator>
        unsigned id(CharIterator begin, CharIterator end) const;
 
        unsigned id(const google::protobuf::Descriptor* desc) const {
            Bitset id_bits;
            dccl::uint32 hardcoded_id = desc->options().GetExtension(dccl::msg).id();
            // pass the hard coded id, that is, (dccl.msg).id,
            // through encode/decode to allow a custom ID codec (if in use)
            // to always take effect.
            id_codec()->field_encode(&id_bits, hardcoded_id, 0);
            std::string id_bytes(id_bits.to_byte_string());
            return id(id_bytes);
        }      
 
        const std::map<int32, const google::protobuf::Descriptor*>& loaded() const { return id2desc_; }
        
            
       
 
 
        void encode(std::string* bytes, const google::protobuf::Message& msg, bool header_only = false, int user_id = -1);
            
        size_t encode(char* bytes, size_t max_len, const google::protobuf::Message& msg, bool header_only = false, int user_id = -1);
 
        template <typename CharIterator>
            CharIterator decode(CharIterator begin, CharIterator end, google::protobuf::Message* msg, bool header_only = false);
 
        void decode(const std::string& bytes, google::protobuf::Message* msg, bool header_only = false);
 
        void decode(std::string* bytes, google::protobuf::Message* msg);
 
        template<typename GoogleProtobufMessagePointer>
            GoogleProtobufMessagePointer decode(const std::string& bytes, bool header_only = false);
        
        template<typename GoogleProtobufMessagePointer>
            GoogleProtobufMessagePointer decode(std::string* bytes);
 
        unsigned size(const google::protobuf::Message& msg, int user_id = -1);
        
        template<typename ProtobufMessage>
            unsigned max_size()
        { return max_size(ProtobufMessage::descriptor()); }            
 
        unsigned max_size(const google::protobuf::Descriptor* desc) const;
 
        template<typename ProtobufMessage>
            unsigned min_size()
        { return min_size(ProtobufMessage::descriptor()); }            
 
        unsigned min_size(const google::protobuf::Descriptor* desc) const;
 
        
 
        
        static std::string default_id_codec_name()
        { return "dccl.default.id"; }        
 
        
        static std::string default_codec_name(int version = 2)
        {
            switch(version)
            {
                case 2:
                    return dccl::DCCLFieldOptions::descriptor()->FindFieldByName("codec")->default_value_string();
                default:
                    return "dccl.default" + boost::lexical_cast<std::string>(version);
            }
            
        }
 
        
      private:
        friend class v2::DefaultMessageCodec;
        Codec(const Codec&);
        Codec& operator= (const Codec&);
 
        void encode_internal(const google::protobuf::Message& msg, bool header_only, Bitset& header_bits, Bitset& body_bits, int user_id);
 
        void encrypt(std::string* s, const std::string& nonce);
        void decrypt(std::string* s, const std::string& nonce);
 
        void set_default_codecs();
 
        boost::shared_ptr<FieldCodecBase> id_codec() const
        {
            return FieldCodecManager::find(google::protobuf::FieldDescriptor::TYPE_UINT32,
                                           id_codec_);
        }
        
      private:
        // SHA256 hash of the crypto passphrase
        std::string crypto_key_;
 
        // strict mode setting
        bool strict_;
        
 // set of DCCL IDs *not* to encrypt        
 std::set<unsigned> skip_crypto_ids_;
 
        // maps `dccl.id`s onto Message Descriptors
        std::map<int32, const google::protobuf::Descriptor*> id2desc_;
        std::string id_codec_;
 
        std::vector<void *> dl_handles_;
        
    };
 
    inline std::ostream& operator<<(std::ostream& os, const Codec& codec)
    {
        codec.info_all(&os);
        return os;
    }
}
 
template<typename GoogleProtobufMessagePointer>
GoogleProtobufMessagePointer dccl::Codec::decode(const std::string& bytes, bool header_only /* = false */)
{
    unsigned this_id = id(bytes);
 
    if(!id2desc_.count(this_id))
        throw(Exception("Message id " + boost::lexical_cast<std::string>(this_id) + " has not been loaded. Call load() before decoding this type."));
                    
    // ownership of this object goes to the caller of decode()
    GoogleProtobufMessagePointer msg =
        dccl::DynamicProtobufManager::new_protobuf_message<GoogleProtobufMessagePointer>(id2desc_.find(this_id)->second);
    decode(bytes, &(*msg), header_only);
    return msg;
}
 
template<typename GoogleProtobufMessagePointer>
GoogleProtobufMessagePointer dccl::Codec::decode(std::string* bytes)
{
    unsigned this_id = id(*bytes);
 
    if(!id2desc_.count(this_id))
        throw(Exception("Message id " + boost::lexical_cast<std::string>(this_id) + " has not been loaded. Call load() before decoding this type."));
                    
    GoogleProtobufMessagePointer msg =
        dccl::DynamicProtobufManager::new_protobuf_message<GoogleProtobufMessagePointer>(id2desc_.find(this_id)->second);
    std::string::iterator new_begin = decode(bytes->begin(), bytes->end(), &(*msg));
    bytes->erase(bytes->begin(), new_begin);
    return msg;
}
 
template<typename CharIterator>
unsigned dccl::Codec::id(CharIterator begin, CharIterator end) const
{
    unsigned id_min_size = 0, id_max_size = 0;
    id_codec()->field_min_size(&id_min_size, 0);
    id_codec()->field_max_size(&id_max_size, 0);
 
    if(std::distance(begin, end) < (id_min_size / BITS_IN_BYTE))
        throw(Exception("Bytes passed (hex: " + hex_encode(begin, end) + ") is too small to be a valid DCCL message"));
 
    Bitset fixed_header_bits;
    fixed_header_bits.from_byte_stream(begin, begin+(size_t)std::ceil(double(id_max_size) / BITS_IN_BYTE));
 
    Bitset these_bits(&fixed_header_bits);
    these_bits.get_more_bits(id_min_size);
 
    boost::any return_value;
    id_codec()->field_decode(&these_bits, &return_value, 0);
 
    return boost::any_cast<uint32>(return_value);
}
 
template <typename CharIterator>
CharIterator dccl::Codec::decode(CharIterator begin, CharIterator end, google::protobuf::Message* msg, bool header_only /*= false*/)
{
    try
    {
        unsigned this_id = id(begin, end);
        
        dlog.is(logger::DEBUG1, logger::DECODE) && dlog  << "Began decoding message of id: " << this_id << std::endl;
        
        if(!id2desc_.count(this_id))
            throw(Exception("Message id " + boost::lexical_cast<std::string>(this_id) + " has not been loaded. Call load() before decoding this type."));
 
        const google::protobuf::Descriptor* desc = msg->GetDescriptor();
 
        dlog.is(logger::DEBUG1, logger::DECODE) && dlog  << "Type name: " << desc->full_name() << std::endl;
 
        boost::shared_ptr<FieldCodecBase> codec = FieldCodecManager::find(desc);
        boost::shared_ptr<internal::FromProtoCppTypeBase> helper = internal::TypeHelper::find(desc);
 
        CharIterator actual_end = end;
        if(codec)
        {
            unsigned head_size_bits;
            unsigned body_size_bits;
            codec->base_max_size(&head_size_bits, desc, HEAD);
            codec->base_max_size(&body_size_bits, desc, BODY);
            unsigned id_size = 0;
            id_codec()->field_size(&id_size, this_id, 0);
            head_size_bits += id_size;
 
            unsigned head_size_bytes = ceil_bits2bytes(head_size_bits);
            unsigned body_size_bytes = ceil_bits2bytes(body_size_bits);
 
            dlog.is(logger::DEBUG2, logger::DECODE) && dlog  << "Head bytes (bits): " << head_size_bytes << "(" << head_size_bits
                                    << "), max body bytes (bits): " << body_size_bytes << "(" << body_size_bits << ")" <<  std::endl;
 
            CharIterator head_bytes_end = begin + head_size_bytes;
            dlog.is(logger::DEBUG3, logger::DECODE) && dlog  << "Unencrypted Head (hex): " << hex_encode(begin, head_bytes_end) << std::endl;
 
            Bitset head_bits;
            head_bits.from_byte_stream(begin, head_bytes_end);
            dlog.is(logger::DEBUG3, logger::DECODE) && dlog  << "Unencrypted Head (bin): " << head_bits << std::endl;
 
            // shift off ID bits
            head_bits >>= id_size;
 
            dlog.is(logger::DEBUG3, logger::DECODE) && dlog  << "Unencrypted Head after ID bits removal (bin): " << head_bits << std::endl;
 
            internal::MessageStack msg_stack;
            msg_stack.push(msg->GetDescriptor());
 
            codec->base_decode(&head_bits, msg, HEAD);
            dlog.is(logger::DEBUG2, logger::DECODE) && dlog  << "after header decode, message is: " << *msg << std::endl;
 
 
            if(header_only)
            {
                dlog.is(logger::DEBUG2, logger::DECODE) && dlog  << "as requested, skipping decrypting and decoding body." << std::endl;
                actual_end = head_bytes_end;
            }
            else
            {
                dlog.is(logger::DEBUG3, logger::DECODE) && dlog  << "Encrypted Body (hex): " << hex_encode(head_bytes_end, end) << std::endl;
 
                Bitset body_bits;
                if(!crypto_key_.empty() && !skip_crypto_ids_.count(this_id))
                {
                    std::string head_bytes(begin, head_bytes_end);
                    std::string body_bytes(head_bytes_end, end);
                    decrypt(&body_bytes, head_bytes);
                    dlog.is(logger::DEBUG3, logger::DECODE) && dlog  << "Unencrypted Body (hex): " << hex_encode(body_bytes) << std::endl;
                    body_bits.from_byte_stream(body_bytes.begin(), body_bytes.end());
                }
                else
                {
                    dlog.is(logger::DEBUG3, logger::DECODE) && dlog  << "Unencrypted Body (hex): " << hex_encode(head_bytes_end, end) << std::endl;
                    body_bits.from_byte_stream(head_bytes_end, end);
                }
 
                dlog.is(logger::DEBUG3, logger::DECODE) && dlog  << "Unencrypted Body (bin): " << body_bits << std::endl;
 
                codec->base_decode(&body_bits, msg, BODY);
                dlog.is(logger::DEBUG2, logger::DECODE) && dlog  << "after header & body decode, message is: " << *msg << std::endl;
 
                actual_end = end - body_bits.size()/BITS_IN_BYTE;
            }
        }
        else
        {
            throw(Exception("Failed to find (dccl.msg).codec `" + desc->options().GetExtension(dccl::msg).codec() + "`"));
        }
 
        dlog.is(logger::DEBUG1, logger::DECODE) && dlog  << "Successfully decoded message of type: " << desc->full_name() << std::endl;
        return actual_end;
    }
    catch(std::exception& e)
    {
        std::stringstream ss;
 
        ss << "Message " << hex_encode(begin, end) <<  " failed to decode. Reason: " << e.what() << std::endl;
 
        dlog.is(logger::DEBUG1, logger::DECODE) && dlog << ss.str() << std::endl;
        throw(Exception(ss.str()));
    }
}
 
#endif