DCCL v4
field_codec_arithmetic.cpp
1 // Copyright 2012-2023:
2 // GobySoft, LLC (2013-)
3 // Massachusetts Institute of Technology (2007-2014)
4 // Community contributors (see AUTHORS file)
5 // File authors:
6 // Toby Schneider <toby@gobysoft.org>
7 //
8 //
9 // This file is part of the Dynamic Compact Control Language Library
10 // ("DCCL").
11 //
12 // DCCL is free software: you can redistribute it and/or modify
13 // it under the terms of the GNU Lesser General Public License as published by
14 // the Free Software Foundation, either version 2.1 of the License, or
15 // (at your option) any later version.
16 //
17 // DCCL is distributed in the hope that it will be useful,
18 // but WITHOUT ANY WARRANTY; without even the implied warranty of
19 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 // GNU Lesser General Public License for more details.
21 //
22 // You should have received a copy of the GNU Lesser General Public License
23 // along with DCCL. If not, see <http://www.gnu.org/licenses/>.
24 #include "field_codec_arithmetic.h"
25 #include "../codec.h"
26 #include "../field_codec_manager.h"
27 
28 using dccl::dlog;
29 using namespace dccl::logger;
30 
31 const dccl::arith::Model::symbol_type dccl::arith::Model::OUT_OF_RANGE_SYMBOL;
32 const dccl::arith::Model::symbol_type dccl::arith::Model::EOF_SYMBOL;
33 const dccl::arith::Model::symbol_type dccl::arith::Model::MIN_SYMBOL;
34 const int dccl::arith::Model::CODE_VALUE_BITS;
35 const int dccl::arith::Model::FREQUENCY_BITS;
36 const dccl::arith::Model::freq_type dccl::arith::Model::MAX_FREQUENCY;
37 
38 #if DCCL_THREAD_SUPPORT
39 std::recursive_mutex dccl::arith::Model::last_bits_map_mutex;
40 #endif
41 std::map<std::string, std::map<std::string, dccl::Bitset>> dccl::arith::Model::last_bits_map;
42 
43 // shared library load
44 extern "C"
45 {
46  void dccl3_load(dccl::Codec* dccl) { dccl_arithmetic_load(dccl); }
47 
48  void dccl_arithmetic_load(dccl::Codec* dccl)
49  {
50  using namespace dccl;
51  using namespace dccl::arith;
52 
53  dccl->manager().add<ArithmeticFieldCodec<int32>>("_arithmetic");
54  dccl->manager().add<ArithmeticFieldCodec<int64>>("_arithmetic");
55  dccl->manager().add<ArithmeticFieldCodec<uint32>>("_arithmetic");
56  dccl->manager().add<ArithmeticFieldCodec<uint64>>("_arithmetic");
57  dccl->manager().add<ArithmeticFieldCodec<double>>("_arithmetic");
58  dccl->manager().add<ArithmeticFieldCodec<float>>("_arithmetic");
59  dccl->manager().add<ArithmeticFieldCodec<bool>>("_arithmetic");
60  dccl->manager().add<ArithmeticFieldCodec<const google::protobuf::EnumValueDescriptor*>>(
61  "_arithmetic");
62 
63  dccl->manager().add<ArithmeticFieldCodec<int32>>("dccl.arithmetic");
64  dccl->manager().add<ArithmeticFieldCodec<int64>>("dccl.arithmetic");
65  dccl->manager().add<ArithmeticFieldCodec<uint32>>("dccl.arithmetic");
66  dccl->manager().add<ArithmeticFieldCodec<uint64>>("dccl.arithmetic");
67  dccl->manager().add<ArithmeticFieldCodec<double>>("dccl.arithmetic");
68  dccl->manager().add<ArithmeticFieldCodec<float>>("dccl.arithmetic");
69  dccl->manager().add<ArithmeticFieldCodec<bool>>("dccl.arithmetic");
70  dccl->manager().add<ArithmeticFieldCodec<const google::protobuf::EnumValueDescriptor*>>(
71  "dccl.arithmetic");
72  }
73  void dccl3_unload(dccl::Codec* dccl) { dccl_arithmetic_unload(dccl); }
74 
75  void dccl_arithmetic_unload(dccl::Codec* dccl)
76  {
77  using namespace dccl;
78  using namespace dccl::arith;
79 
80  dccl->manager().remove<ArithmeticFieldCodec<int32>>("_arithmetic");
81  dccl->manager().remove<ArithmeticFieldCodec<int64>>("_arithmetic");
82  dccl->manager().remove<ArithmeticFieldCodec<uint32>>("_arithmetic");
83  dccl->manager().remove<ArithmeticFieldCodec<uint64>>("_arithmetic");
84  dccl->manager().remove<ArithmeticFieldCodec<double>>("_arithmetic");
85  dccl->manager().remove<ArithmeticFieldCodec<float>>("_arithmetic");
86  dccl->manager().remove<ArithmeticFieldCodec<bool>>("_arithmetic");
87  dccl->manager().remove<ArithmeticFieldCodec<const google::protobuf::EnumValueDescriptor*>>(
88  "_arithmetic");
89 
90  dccl->manager().remove<ArithmeticFieldCodec<int32>>("dccl.arithmetic");
91  dccl->manager().remove<ArithmeticFieldCodec<int64>>("dccl.arithmetic");
92  dccl->manager().remove<ArithmeticFieldCodec<uint32>>("dccl.arithmetic");
93  dccl->manager().remove<ArithmeticFieldCodec<uint64>>("dccl.arithmetic");
94  dccl->manager().remove<ArithmeticFieldCodec<double>>("dccl.arithmetic");
95  dccl->manager().remove<ArithmeticFieldCodec<float>>("dccl.arithmetic");
96  dccl->manager().remove<ArithmeticFieldCodec<bool>>("dccl.arithmetic");
97  dccl->manager().remove<ArithmeticFieldCodec<const google::protobuf::EnumValueDescriptor*>>(
98  "dccl.arithmetic");
99  }
100 }
101 
102 dccl::arith::Model::symbol_type dccl::arith::Model::value_to_symbol(value_type value) const
103 {
104  if (value < *user_model_.value_bound().begin() ||
105  value > *(user_model_.value_bound().end() - 1))
106  return Model::OUT_OF_RANGE_SYMBOL;
107 
108  google::protobuf::RepeatedField<double>::const_iterator upper_it =
109  std::upper_bound(user_model_.value_bound().begin(), user_model_.value_bound().end(), value);
110 
111  google::protobuf::RepeatedField<double>::const_iterator lower_it =
112  (upper_it == user_model_.value_bound().begin()) ? upper_it : upper_it - 1;
113 
114  double lower_diff = std::abs((*lower_it) * (*lower_it) - value * value);
115  double upper_diff = std::abs((*upper_it) * (*upper_it) - value * value);
116 
117  // std::cout << "value: " << value << std::endl;
118  // std::cout << "lower_value: " << *lower_it << std::endl;
119  // std::cout << "upper_value: " << *upper_it << std::endl;
120 
121  symbol_type symbol =
122  ((lower_diff < upper_diff) ? lower_it : upper_it) - user_model_.value_bound().begin();
123 
124  return symbol;
125 }
126 
127 dccl::arith::Model::value_type dccl::arith::Model::symbol_to_value(symbol_type symbol) const
128 {
129  if (symbol == EOF_SYMBOL)
130  throw(Exception("EOF symbol has no value."));
131 
132  value_type value = (symbol == Model::OUT_OF_RANGE_SYMBOL)
133  ? std::numeric_limits<value_type>::quiet_NaN()
134  : user_model_.value_bound(symbol);
135 
136  return value;
137 }
138 
139 std::pair<dccl::arith::Model::freq_type, dccl::arith::Model::freq_type>
140 dccl::arith::Model::symbol_to_cumulative_freq(symbol_type symbol, ModelState state) const
141 {
142  const auto& c_freqs =
143  (state == ENCODER) ? encoder_cumulative_freqs_ : decoder_cumulative_freqs_;
144 
145  auto c_freq_it = c_freqs.find(symbol);
146  std::pair<freq_type, freq_type> c_freq_range;
147  c_freq_range.second = c_freq_it->second;
148  if (c_freq_it == c_freqs.begin())
149  {
150  c_freq_range.first = 0;
151  }
152  else
153  {
154  c_freq_it--;
155  c_freq_range.first = c_freq_it->second;
156  }
157  return c_freq_range;
158 }
159 
160 std::pair<dccl::arith::Model::symbol_type, dccl::arith::Model::symbol_type>
161 dccl::arith::Model::cumulative_freq_to_symbol(std::pair<freq_type, freq_type> c_freq_pair,
162  ModelState state) const
163 {
164  const auto& c_freqs =
165  (state == ENCODER) ? encoder_cumulative_freqs_ : decoder_cumulative_freqs_;
166 
167  std::pair<symbol_type, symbol_type> symbol_pair;
168 
169  // find the symbol for which the cumulative frequency is greater than
170  // e.g.
171  // symbol: 0 freq: 10 c_freq: 10 [0 ... 10)
172  // symbol: 1 freq: 15 c_freq: 25 [10 ... 25)
173  // symbol: 2 freq: 10 c_freq: 35 [25 ... 35)
174  // searching for c_freq of 30 should return symbol 2
175  // searching for c_freq of 10 should return symbol 1
176  auto search = c_freq_pair.first;
177  for (const auto& p : c_freqs)
178  {
179  if (search < p.second)
180  {
181  symbol_pair.first = p.first;
182  break;
183  }
184  }
185 
186  if (symbol_pair.first == c_freqs.rbegin()->first)
187  symbol_pair.second = symbol_pair.first; // last symbol can't be ambiguous on the low end
188  else if (c_freqs.find(symbol_pair.first)->second > c_freq_pair.second)
189  symbol_pair.second = symbol_pair.first; // unambiguously this symbol
190  else
191  symbol_pair.second = symbol_pair.first + 1;
192 
193  return symbol_pair;
194 }
195 
196 void dccl::arith::Model::update_model(symbol_type symbol, ModelState state)
197 {
198  if (!user_model_.is_adaptive())
199  return;
200 
201  auto& c_freqs = (state == ENCODER) ? encoder_cumulative_freqs_ : decoder_cumulative_freqs_;
202 
203  if (dlog.check(DEBUG3))
204  {
205  dlog.is(DEBUG3) && dlog << "Model was: " << std::endl;
206  for (symbol_type i = MIN_SYMBOL, n = max_symbol(); i <= n; ++i)
207  {
208  auto it = c_freqs.find(i);
209  if (it != c_freqs.end())
210  dlog.is(DEBUG3) && dlog << "Symbol: " << it->first << ", c_freq: " << it->second
211  << std::endl;
212  }
213  }
214 
215  for (symbol_type i = max_symbol(), n = symbol; i >= n; --i)
216  {
217  auto it = c_freqs.find(i);
218  if (it != c_freqs.end())
219  ++it->second;
220  }
221 
222  if (dlog.check(DEBUG3))
223  {
224  dlog.is(DEBUG3) && dlog << "Model is now: " << std::endl;
225  for (symbol_type i = MIN_SYMBOL, n = max_symbol(); i <= n; ++i)
226  {
227  auto it = c_freqs.find(i);
228  if (it != c_freqs.end())
229  dlog.is(DEBUG3) && dlog << "Symbol: " << it->first << ", c_freq: " << it->second
230  << std::endl;
231  }
232  }
233 
234  dlog.is(DEBUG3) && dlog << "total freq: " << total_freq(state) << std::endl;
235 }
236 
237 void dccl::arith::ModelManager::set_model(dccl::Codec& codec,
238  const protobuf::ArithmeticModel& model)
239 {
240  model_manager(codec.manager())._set_model(model);
241 }
242 
243 dccl::arith::ModelManager& dccl::arith::model_manager(FieldCodecManagerLocal& manager)
244 {
245  if (!manager.codec_data().template has_codec_specific_data<ArithmeticFieldCodecBase<>>())
246  {
247  auto model_manager = std::make_shared<dccl::any>(ModelManager());
248  manager.codec_data().template set_codec_specific_data<ArithmeticFieldCodecBase<>>(
249  model_manager);
250  }
251  return dccl::any_cast<ModelManager&>(
252  *manager.codec_data().template codec_specific_data<ArithmeticFieldCodecBase<>>());
253 }
dccl::FieldCodecManagerLocal
A class for managing the various field codecs. Here you can add and remove field codecs....
Definition: field_codec_manager.h:39
dccl::arith::ArithmeticFieldCodecBase
Definition: field_codec_arithmetic.h:214
dccl
Dynamic Compact Control Language namespace.
Definition: any.h:46
dccl::Logger::is
bool is(logger::Verbosity verbosity, logger::Group group=logger::GENERAL)
Indicates the verbosity of the Logger until the next std::flush or std::endl. The boolean return is u...
Definition: logger.h:192
dccl::Exception
Exception class for DCCL.
Definition: exception.h:46
dccl::arith::ModelManager
Definition: field_codec_arithmetic.h:128
dccl::Codec
The Dynamic CCL enCODer/DECoder. This is the main class you will use to load, encode and decode DCCL ...
Definition: codec.h:62
dccl::arith::ArithmeticFieldCodec< const google::protobuf::EnumValueDescriptor * >
Definition: field_codec_arithmetic.h:713
dccl::Logger::check
bool check(logger::Verbosity verbosity)
Same as is() but doesn't set the verbosity or lock the mutex.
Definition: logger.h:181
dccl::arith::protobuf::ArithmeticModel
Definition: arithmetic.pb.h:73
dccl::arith::ArithmeticFieldCodec
Definition: field_codec_arithmetic.h:699
dccl::arith
DCCL Arithmetic Encoder Library namespace.
Definition: field_codec_arithmetic.h:54