DCCL v4
test.cpp
1 // Copyright 2012-2023:
2 // GobySoft, LLC (2013-)
3 // Massachusetts Institute of Technology (2007-2014)
4 // Community contributors (see AUTHORS file)
5 // File authors:
6 // Toby Schneider <toby@gobysoft.org>
7 // Chris Murphy <cmurphy@aphysci.com>
8 //
9 //
10 // This file is part of the Dynamic Compact Control Language Library
11 // ("DCCL").
12 //
13 // DCCL is free software: you can redistribute it and/or modify
14 // it under the terms of the GNU Lesser General Public License as published by
15 // the Free Software Foundation, either version 2.1 of the License, or
16 // (at your option) any later version.
17 //
18 // DCCL is distributed in the hope that it will be useful,
19 // but WITHOUT ANY WARRANTY; without even the implied warranty of
20 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 // GNU Lesser General Public License for more details.
22 //
23 // You should have received a copy of the GNU Lesser General Public License
24 // along with DCCL. If not, see <http://www.gnu.org/licenses/>.
25 // tests arithmetic encoder
26 
27 #include <google/protobuf/descriptor.pb.h>
28 
29 #include "../../arithmetic/field_codec_arithmetic.h"
30 #include "../../codec.h"
31 
32 #include "test_arithmetic.pb.h"
33 
34 #include "../../binary.h"
35 using namespace dccl::test::arith;
36 
37 
38 void run_test(dccl::arith::protobuf::ArithmeticModel& model,
39  const google::protobuf::Message& msg_in, bool set_model = true)
40 {
41  static int i = 0;
42 
43  static dccl::Codec codec;
44 
45  if (!i)
46  {
47  void* dl_handle = dlopen(DCCL_ARITHMETIC_NAME, RTLD_LAZY);
48  if (!dl_handle)
49  {
50  std::cerr << "Failed to open " << DCCL_ARITHMETIC_NAME << std::endl;
51  exit(1);
52  }
53  codec.load_library(dl_handle);
54  }
55 
56  if (set_model)
57  {
58  model.set_name("model");
59  dccl::arith::ModelManager::set_model(codec, model);
60  }
61 
62  codec.info(msg_in.GetDescriptor(), &std::cout);
63 
64  codec.load(msg_in.GetDescriptor());
65 
66  std::cout << "Message in:\n" << msg_in.DebugString() << std::endl;
67 
68  std::cout << "Try encode..." << std::endl;
69  std::string bytes;
70  codec.encode(&bytes, msg_in);
71  std::cout << "... got bytes (hex): " << dccl::hex_encode(bytes) << std::endl;
72 
73  std::cout << "Try decode..." << std::endl;
74 
75  std::shared_ptr<google::protobuf::Message> msg_out(msg_in.New());
76  codec.decode(bytes, msg_out.get());
77 
78  std::cout << "... got Message out:\n" << msg_out->DebugString() << std::endl;
79 
80  assert(msg_in.SerializeAsString() == msg_out->SerializeAsString());
81  ++i;
82 }
83 
84 // usage: dccl_test10 [boolean: verbose]
85 int main(int argc, char* argv[])
86 {
87  if (argc > 1 && std::string(argv[1]) == "1")
88  dccl::dlog.connect(dccl::logger::DEBUG3_PLUS, &std::cerr);
89  else
90  dccl::dlog.connect(dccl::logger::DEBUG2_PLUS, &std::cerr);
91 
92  dccl::Codec codec;
93 
94  // test case from Practical Implementations of Arithmetic Coding by Paul G. Howard and Je rey Scott Vitter
95  {
96  dccl::arith::protobuf::ArithmeticModel model;
97 
98  model.set_eof_frequency(4); // "a"
99 
100  model.add_value_bound(0);
101  model.add_frequency(5); // "b"
102 
103  model.add_value_bound(1);
104  model.add_frequency(1); // "EOF"
105 
106  model.add_value_bound(2);
107 
108  model.set_out_of_range_frequency(0);
109 
110  ArithmeticDoubleTestMsg msg_in;
111 
112  msg_in.add_value(0); // b
113  msg_in.add_value(0); // b
114  msg_in.add_value(0); // b
115  msg_in.add_value(1); // "EOF"
116 
117  run_test(model, msg_in);
118  }
119 
120  // misc test case
121  {
122  dccl::arith::protobuf::ArithmeticModel model;
123 
124  model.add_value_bound(100.0);
125  model.add_frequency(100);
126 
127  model.add_value_bound(100.1);
128  model.add_frequency(100);
129 
130  model.add_value_bound(100.2);
131  model.add_frequency(100);
132 
133  model.add_value_bound(100.3);
134  model.add_frequency(100);
135 
136  model.add_value_bound(100.4);
137  model.add_frequency(90);
138 
139  model.add_value_bound(100.5);
140  model.add_frequency(125);
141 
142  model.add_value_bound(100.6);
143  model.add_frequency(125);
144 
145  model.add_value_bound(100.7);
146  model.add_frequency(125);
147 
148  model.add_value_bound(100.8);
149 
150  model.set_eof_frequency(25);
151  model.set_out_of_range_frequency(10);
152 
153  ArithmeticDoubleTestMsg msg_in;
154 
155  msg_in.add_value(100.5);
156  msg_in.add_value(100.7);
157  msg_in.add_value(100.2);
158 
159  run_test(model, msg_in);
160  }
161 
162  // edge case 1, should be just a single bit ("1")
163  {
164  dccl::arith::protobuf::ArithmeticModel model;
165 
166  model.set_eof_frequency(10);
167  model.set_out_of_range_frequency(0);
168 
169  model.add_value_bound(1);
170  model.add_frequency(2);
171 
172  model.add_value_bound(2);
173  model.add_frequency(3);
174 
175  model.add_value_bound(3);
176  model.add_frequency(85);
177 
178  model.add_value_bound(4);
179 
180  ArithmeticEnumTestMsg msg_in;
181 
182  msg_in.add_value(ENUM_C);
183  msg_in.add_value(ENUM_C);
184  msg_in.add_value(ENUM_C);
185  msg_in.add_value(ENUM_C);
186 
187  run_test(model, msg_in);
188  }
189 
190  // edge case 2, should be full 23 or 24 bits
191  {
192  dccl::arith::protobuf::ArithmeticModel model;
193 
194  model.set_eof_frequency(10);
195  model.set_out_of_range_frequency(0);
196 
197  model.add_value_bound(1);
198  model.add_frequency(2);
199 
200  model.add_value_bound(2);
201  model.add_frequency(3);
202 
203  model.add_value_bound(3);
204  model.add_frequency(85);
205 
206  model.add_value_bound(4);
207 
208  ArithmeticEnumTestMsg msg_in;
209 
210  msg_in.add_value(ENUM_A);
211  msg_in.add_value(ENUM_A);
212  msg_in.add_value(ENUM_A);
213  msg_in.add_value(ENUM_A);
214 
215  run_test(model, msg_in);
216  }
217 
218  {
219  dccl::arith::protobuf::ArithmeticModel model;
220 
221  model.set_eof_frequency(10);
222  model.set_out_of_range_frequency(0);
223 
224  model.add_value_bound(1);
225  model.add_frequency(2);
226 
227  model.add_value_bound(2);
228  model.add_frequency(3);
229 
230  model.add_value_bound(3);
231  model.add_frequency(85);
232 
233  model.add_value_bound(4);
234 
235  ArithmeticSingleEnumTestMsg msg_in;
236 
237  msg_in.set_value(ENUM_B);
238 
239  run_test(model, msg_in);
240  }
241 
242  // test case from Practical Implementations of Arithmetic Coding by Paul G. Howard and Je rey Scott Vitter
243  {
244  dccl::arith::protobuf::ArithmeticModel model;
245 
246  model.set_eof_frequency(1);
247 
248  model.add_value_bound(0);
249  model.add_frequency(1);
250 
251  model.add_value_bound(1);
252  model.add_frequency(1);
253 
254  model.add_value_bound(2);
255 
256  model.set_out_of_range_frequency(1);
257 
258  ArithmeticDouble3TestMsg msg_in;
259 
260  msg_in.add_value(0);
261  msg_in.add_value(0);
262  msg_in.add_value(0);
263  msg_in.add_value(1);
264 
265  model.set_is_adaptive(true);
266  run_test(model, msg_in);
267  run_test(model, msg_in, false);
268  run_test(model, msg_in, false);
269  run_test(model, msg_in, false);
270  }
271 
272  // test case from Arithmetic Coding revealed: A guided tour from theory to praxis Sable Technical Report No. 2007-5 Eric Bodden
273 
274  {
275  dccl::arith::protobuf::ArithmeticModel model;
276 
277  model.set_eof_frequency(0);
278  model.set_out_of_range_frequency(0);
279 
280  model.add_value_bound(1);
281  model.add_frequency(2);
282 
283  model.add_value_bound(2);
284  model.add_frequency(1);
285 
286  model.add_value_bound(3);
287  model.add_frequency(3);
288 
289  model.add_value_bound(4);
290  model.add_frequency(1);
291 
292  model.add_value_bound(5);
293  model.add_frequency(1);
294 
295  model.add_value_bound(6);
296 
297  ArithmeticEnum2TestMsg msg_in;
298 
299  msg_in.add_value(ENUM2_A);
300  msg_in.add_value(ENUM2_B);
301  msg_in.add_value(ENUM2_C);
302  msg_in.add_value(ENUM2_C);
303  msg_in.add_value(ENUM2_E);
304  msg_in.add_value(ENUM2_D);
305  msg_in.add_value(ENUM2_A);
306  msg_in.add_value(ENUM2_C);
307 
308  run_test(model, msg_in);
309  }
310 
311  // randomly generate a model and a message
312  // loop over all message lengths from 0 to 100
313  srand(time(nullptr));
314  for (unsigned i = 0; i <= ArithmeticDouble2TestMsg::descriptor()
315  ->FindFieldByName("value")
316  ->options()
317  .GetExtension(dccl::field)
318  .max_repeat();
319  ++i)
320  {
321  dccl::arith::protobuf::ArithmeticModel model;
322 
323  // pick some endpoints
324  dccl::int32 low = -(rand() % std::numeric_limits<dccl::int32>::max());
325  dccl::int32 high = rand() % std::numeric_limits<dccl::int32>::max();
326 
327  std::cout << "low: " << low << ", high: " << high << std::endl;
328 
329  // number of symbols
330  dccl::int32 symbols = rand() % 1000 + 10;
331 
332  std::cout << "symbols: " << symbols << std::endl;
333 
334  // maximum freq
335  dccl::arith::Model::freq_type each_max_freq =
336  dccl::arith::Model::MAX_FREQUENCY / (symbols + 2);
337 
338  std::cout << "each_max_freq: " << each_max_freq << std::endl;
339 
340  model.set_eof_frequency(rand() % each_max_freq + 1);
341  model.set_out_of_range_frequency(rand() % each_max_freq + 1);
342 
343  model.add_value_bound(low);
344  model.add_frequency(rand() % each_max_freq + 1);
345  for (int j = 1; j < symbols; ++j)
346  {
347  // std::cout << "j: " << j << std::endl;
348 
349  dccl::int32 remaining_range = high - model.value_bound(j - 1);
350  model.add_value_bound(model.value_bound(j - 1) +
351  rand() % (remaining_range / symbols - j) + 1);
352  model.add_frequency(rand() % each_max_freq + 1);
353  }
354 
355  model.add_value_bound(high);
356 
357  ArithmeticDouble2TestMsg msg_in;
358 
359  for (unsigned j = 0; j < i; ++j) msg_in.add_value(model.value_bound(rand() % symbols));
360 
361  run_test(model, msg_in);
362 
363  std::cout << "end random test #" << i << std::endl;
364  }
365 
366  std::cout << "all tests passed" << std::endl;
367 }
dccl::Codec
The Dynamic CCL enCODer/DECoder. This is the main class you will use to load, encode and decode DCCL ...
Definition: codec.h:62
dccl::Logger::connect
void connect(int verbosity_mask, Slot slot)
Connect the output of one or more given verbosities to a slot (function pointer or similar)
Definition: logger.h:214
dccl::hex_encode
void hex_encode(CharIterator begin, CharIterator end, std::string *out, bool upper_case=false)
Encodes a (little-endian) hexadecimal string from a byte string. Index 0 of begin is written to index...
Definition: binary.h:100
dccl::arith::protobuf::ArithmeticModel
Definition: arithmetic.pb.h:73
Message
dccl::int32
google::protobuf::int32 int32
a signed 32 bit integer
Definition: common.h:58