DCCL v5
Loading...
Searching...
No Matches
test.cpp
1// Copyright 2012-2023:
2// GobySoft, LLC (2013-)
3// Massachusetts Institute of Technology (2007-2014)
4// Community contributors (see AUTHORS file)
5// File authors:
6// Toby Schneider <toby@gobysoft.org>
7// Chris Murphy <cmurphy@aphysci.com>
8//
9//
10// This file is part of the Dynamic Compact Control Language Library
11// ("DCCL").
12//
13// DCCL is free software: you can redistribute it and/or modify
14// it under the terms of the GNU Lesser General Public License as published by
15// the Free Software Foundation, either version 2.1 of the License, or
16// (at your option) any later version.
17//
18// DCCL is distributed in the hope that it will be useful,
19// but WITHOUT ANY WARRANTY; without even the implied warranty of
20// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21// GNU Lesser General Public License for more details.
22//
23// You should have received a copy of the GNU Lesser General Public License
24// along with DCCL. If not, see <http://www.gnu.org/licenses/>.
25// tests arithmetic encoder
26
27#include "../../arithmetic/field_codec_arithmetic.h"
28#include "../../codec.h"
29
30#include "test_arithmetic.pb.h"
31
32#include "../../binary.h"
33using namespace dccl::test::arith;
34
35void run_test(dccl::arith::protobuf::ArithmeticModel& model,
36 const google::protobuf::Message& msg_in, bool set_model = true)
37{
38 static int i = 0;
39
40 static dccl::Codec codec;
41
42 if (!i)
43 {
44 void* dl_handle = dlopen(DCCL_ARITHMETIC_NAME, RTLD_LAZY);
45 if (!dl_handle)
46 {
47 std::cerr << "Failed to open " << DCCL_ARITHMETIC_NAME << std::endl;
48 exit(1);
49 }
50 codec.load_library(dl_handle);
51 }
52
53 if (set_model)
54 {
55 model.set_name("model");
56 dccl::arith::ModelManager::set_model(codec, model);
57 }
58
59 if (dccl::dlog.is(dccl::logger::INFO))
60 codec.info(msg_in.GetDescriptor(), &dccl::dlog);
61
62 codec.load(msg_in.GetDescriptor());
63
64 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "Message in:\n"
65 << msg_in.DebugString() << std::endl;
66
67 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "Try encode..." << std::endl;
68 std::string bytes;
69 codec.encode(&bytes, msg_in);
70 dccl::dlog.is(dccl::logger::INFO) &&
71 dccl::dlog << "... got bytes (hex): " << dccl::hex_encode(bytes) << std::endl;
72
73 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "Try decode..." << std::endl;
74
75 std::shared_ptr<google::protobuf::Message> msg_out(msg_in.New());
76 codec.decode(bytes, msg_out.get());
77
78 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "... got Message out:\n"
79 << msg_out->DebugString() << std::endl;
80
81 assert(msg_in.SerializeAsString() == msg_out->SerializeAsString());
82 ++i;
83}
84
85// Runs one repeated field of the given message type through the arithmetic
86// codec. Values are kept to 0 and 1 so the same model and body work for every
87// field type, bool included.
88template <typename Msg> void run_scalar_type_test(dccl::arith::protobuf::ArithmeticModel& model)
89{
90 Msg msg_in;
91 msg_in.add_value(0);
92 msg_in.add_value(1);
93 run_test(model, msg_in);
94}
95
96// usage: dccl_test10 [-v | 1]
97int main(int argc, char* argv[])
98{
99 bool verbose = false;
100 bool legacy_verbose = false;
101 for (int i = 1; i < argc; ++i)
102 {
103 if (argv[i] && argv[i][0] == '-' && argv[i][1] == 'v' && argv[i][2] == '\0')
104 verbose = true;
105 else if (argv[i] && std::string(argv[i]) == "1")
106 legacy_verbose = true;
107 }
108
109 if (verbose)
110 dccl::dlog.connect(dccl::logger::ALL, &std::cerr);
111 else if (legacy_verbose)
112 dccl::dlog.connect(dccl::logger::DEBUG3_PLUS, &std::cerr);
113 else
114 dccl::dlog.connect(dccl::logger::WARN_PLUS, &std::cerr);
115
116 dccl::Codec codec;
117
118 // test case from Practical Implementations of Arithmetic Coding by Paul G. Howard and Je rey Scott Vitter
119 {
120 dccl::arith::protobuf::ArithmeticModel model;
121
122 model.set_eof_frequency(4); // "a"
123
124 model.add_value_bound(0);
125 model.add_frequency(5); // "b"
126
127 model.add_value_bound(1);
128 model.add_frequency(1); // "EOF"
129
130 model.add_value_bound(2);
131
132 model.set_out_of_range_frequency(0);
133
134 ArithmeticDoubleTestMsg msg_in;
135
136 msg_in.add_value(0); // b
137 msg_in.add_value(0); // b
138 msg_in.add_value(0); // b
139 msg_in.add_value(1); // "EOF"
140
141 run_test(model, msg_in);
142 }
143
144 // misc test case
145 {
146 dccl::arith::protobuf::ArithmeticModel model;
147
148 model.add_value_bound(100.0);
149 model.add_frequency(100);
150
151 model.add_value_bound(100.1);
152 model.add_frequency(100);
153
154 model.add_value_bound(100.2);
155 model.add_frequency(100);
156
157 model.add_value_bound(100.3);
158 model.add_frequency(100);
159
160 model.add_value_bound(100.4);
161 model.add_frequency(90);
162
163 model.add_value_bound(100.5);
164 model.add_frequency(125);
165
166 model.add_value_bound(100.6);
167 model.add_frequency(125);
168
169 model.add_value_bound(100.7);
170 model.add_frequency(125);
171
172 model.add_value_bound(100.8);
173
174 model.set_eof_frequency(25);
175 model.set_out_of_range_frequency(10);
176
177 ArithmeticDoubleTestMsg msg_in;
178
179 msg_in.add_value(100.5);
180 msg_in.add_value(100.7);
181 msg_in.add_value(100.2);
182
183 run_test(model, msg_in);
184 }
185
186 // edge case 1, should be just a single bit ("1")
187 {
188 dccl::arith::protobuf::ArithmeticModel model;
189
190 model.set_eof_frequency(10);
191 model.set_out_of_range_frequency(0);
192
193 model.add_value_bound(1);
194 model.add_frequency(2);
195
196 model.add_value_bound(2);
197 model.add_frequency(3);
198
199 model.add_value_bound(3);
200 model.add_frequency(85);
201
202 model.add_value_bound(4);
203
204 ArithmeticEnumTestMsg msg_in;
205
206 msg_in.add_value(ENUM_C);
207 msg_in.add_value(ENUM_C);
208 msg_in.add_value(ENUM_C);
209 msg_in.add_value(ENUM_C);
210
211 run_test(model, msg_in);
212 }
213
214 // edge case 2, should be full 23 or 24 bits
215 {
216 dccl::arith::protobuf::ArithmeticModel model;
217
218 model.set_eof_frequency(10);
219 model.set_out_of_range_frequency(0);
220
221 model.add_value_bound(1);
222 model.add_frequency(2);
223
224 model.add_value_bound(2);
225 model.add_frequency(3);
226
227 model.add_value_bound(3);
228 model.add_frequency(85);
229
230 model.add_value_bound(4);
231
232 ArithmeticEnumTestMsg msg_in;
233
234 msg_in.add_value(ENUM_A);
235 msg_in.add_value(ENUM_A);
236 msg_in.add_value(ENUM_A);
237 msg_in.add_value(ENUM_A);
238
239 run_test(model, msg_in);
240 }
241
242 {
243 dccl::arith::protobuf::ArithmeticModel model;
244
245 model.set_eof_frequency(10);
246 model.set_out_of_range_frequency(0);
247
248 model.add_value_bound(1);
249 model.add_frequency(2);
250
251 model.add_value_bound(2);
252 model.add_frequency(3);
253
254 model.add_value_bound(3);
255 model.add_frequency(85);
256
257 model.add_value_bound(4);
258
259 ArithmeticSingleEnumTestMsg msg_in;
260
261 msg_in.set_value(ENUM_B);
262
263 run_test(model, msg_in);
264 }
265
266 // test case from Practical Implementations of Arithmetic Coding by Paul G. Howard and Je rey Scott Vitter
267 {
268 dccl::arith::protobuf::ArithmeticModel model;
269
270 model.set_eof_frequency(1);
271
272 model.add_value_bound(0);
273 model.add_frequency(1);
274
275 model.add_value_bound(1);
276 model.add_frequency(1);
277
278 model.add_value_bound(2);
279
280 model.set_out_of_range_frequency(1);
281
282 ArithmeticDouble3TestMsg msg_in;
283
284 msg_in.add_value(0);
285 msg_in.add_value(0);
286 msg_in.add_value(0);
287 msg_in.add_value(1);
288
289 model.set_is_adaptive(true);
290 run_test(model, msg_in);
291 run_test(model, msg_in, false);
292 run_test(model, msg_in, false);
293 run_test(model, msg_in, false);
294 }
295
296 // test case from Arithmetic Coding revealed: A guided tour from theory to praxis Sable Technical Report No. 2007-5 Eric Bodden
297
298 {
299 dccl::arith::protobuf::ArithmeticModel model;
300
301 model.set_eof_frequency(0);
302 model.set_out_of_range_frequency(0);
303
304 model.add_value_bound(1);
305 model.add_frequency(2);
306
307 model.add_value_bound(2);
308 model.add_frequency(1);
309
310 model.add_value_bound(3);
311 model.add_frequency(3);
312
313 model.add_value_bound(4);
314 model.add_frequency(1);
315
316 model.add_value_bound(5);
317 model.add_frequency(1);
318
319 model.add_value_bound(6);
320
321 ArithmeticEnum2TestMsg msg_in;
322
323 msg_in.add_value(ENUM2_A);
324 msg_in.add_value(ENUM2_B);
325 msg_in.add_value(ENUM2_C);
326 msg_in.add_value(ENUM2_C);
327 msg_in.add_value(ENUM2_E);
328 msg_in.add_value(ENUM2_D);
329 msg_in.add_value(ENUM2_A);
330 msg_in.add_value(ENUM2_C);
331
332 run_test(model, msg_in);
333 }
334
335 // randomly generate a model and a message
336 // loop over all message lengths from 0 to 100
337 srand(time(nullptr));
338 for (unsigned i = 0; i <= ArithmeticDouble2TestMsg::descriptor()
339 ->FindFieldByName("value")
340 ->options()
341 .GetExtension(dccl::field)
342 .max_repeat();
343 ++i)
344 {
345 dccl::arith::protobuf::ArithmeticModel model;
346
347 // pick some endpoints
348 dccl::int32 low = -(rand() % std::numeric_limits<dccl::int32>::max());
349 dccl::int32 high = rand() % std::numeric_limits<dccl::int32>::max();
350
351 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "low: " << low << ", high: " << high
352 << std::endl;
353
354 // number of symbols
355 dccl::int32 symbols = rand() % 1000 + 10;
356
357 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "symbols: " << symbols << std::endl;
358
359 // maximum freq
360 dccl::arith::Model::freq_type each_max_freq =
361 dccl::arith::Model::MAX_FREQUENCY / (symbols + 2);
362
363 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "each_max_freq: " << each_max_freq
364 << std::endl;
365
366 model.set_eof_frequency(rand() % each_max_freq + 1);
367 model.set_out_of_range_frequency(rand() % each_max_freq + 1);
368
369 model.add_value_bound(low);
370 model.add_frequency(rand() % each_max_freq + 1);
371 for (int j = 1; j < symbols; ++j)
372 {
373 // dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "j: " << j << std::endl;
374
375 dccl::int32 remaining_range = high - model.value_bound(j - 1);
376 model.add_value_bound(model.value_bound(j - 1) +
377 rand() % (remaining_range / symbols - j) + 1);
378 model.add_frequency(rand() % each_max_freq + 1);
379 }
380
381 model.add_value_bound(high);
382
383 ArithmeticDouble2TestMsg msg_in;
384
385 for (unsigned j = 0; j < i; ++j) msg_in.add_value(model.value_bound(rand() % symbols));
386
387 run_test(model, msg_in);
388
389 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "end random test #" << i << std::endl;
390 }
391
392 // Every remaining field type the arithmetic codec is registered for. Only
393 // int32, double and enums were exercised before, so the repeated paths for
394 // these types were never run.
395 {
396 dccl::arith::protobuf::ArithmeticModel model;
397 model.set_name("model");
398 model.set_eof_frequency(10);
399 model.set_out_of_range_frequency(0);
400 model.add_value_bound(0);
401 model.add_frequency(10);
402 model.add_value_bound(1);
403 model.add_frequency(10);
404 model.add_value_bound(2);
405
406 run_scalar_type_test<ArithmeticInt64TestMsg>(model);
407 run_scalar_type_test<ArithmeticUInt32TestMsg>(model);
408 run_scalar_type_test<ArithmeticUInt64TestMsg>(model);
409 run_scalar_type_test<ArithmeticFloatTestMsg>(model);
410 run_scalar_type_test<ArithmeticBoolTestMsg>(model);
411 }
412
413 // Loading and then unloading the shared library, which reaches
414 // dccl_arithmetic_unload(); nothing else calls it. A message needing the
415 // arithmetic codec must encode while it is loaded and fail once it is not.
416 {
417 void* dl_handle = dlopen(DCCL_ARITHMETIC_NAME, RTLD_LAZY);
418 assert(dl_handle);
419
420 dccl::Codec unload_codec;
421 unload_codec.load_library(dl_handle);
422
423 dccl::arith::protobuf::ArithmeticModel model;
424 model.set_name("model");
425 model.set_eof_frequency(10);
426 model.set_out_of_range_frequency(0);
427 model.add_value_bound(1);
428 model.add_frequency(2);
429 model.add_value_bound(2);
430 model.add_frequency(3);
431 model.add_value_bound(3);
432 model.add_frequency(85);
433 model.add_value_bound(4);
434 dccl::arith::ModelManager::set_model(unload_codec, model);
435
436 ArithmeticEnumTestMsg msg;
437 msg.add_value(ENUM_A);
438 unload_codec.load(msg.GetDescriptor());
439
440 std::string bytes;
441 unload_codec.encode(&bytes, msg);
442 assert(!bytes.empty());
443
444 unload_codec.unload_library(dl_handle);
445
446 // with the codec gone, the same message can no longer be loaded
447 dccl::Codec after_unload;
448 bool threw = false;
449 try
450 {
451 after_unload.load(msg.GetDescriptor());
452 }
453 catch (const dccl::Exception&)
454 {
455 threw = true;
456 }
457 assert(threw);
458
459 dlclose(dl_handle);
460 }
461
462 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "all tests passed" << std::endl;
463}
dccl::Codec
The Dynamic CCL enCODer/DECoder. This is the main class you will use to load, encode and decode DCCL ...
Definition codec.h:61
dccl::Codec::unload_library
void unload_library(void *dl_handle)
Remove codecs and/or unload messages present in the given shared library handle.
Definition codec.cpp:710
dccl::Codec::encode
void encode(std::string *bytes, const google::protobuf::Message &msg, bool header_only=false, int user_id=-1)
Encodes a DCCL message.
Definition codec.cpp:306
dccl::Codec::load_library
void load_library(void *dl_handle)
Add codecs and/or load messages present in the given shared library handle.
Definition codec.cpp:698
dccl::Codec::load
std::size_t load()
All messages must be explicited loaded and validated (size checks, option extensions checks,...
Definition codec.h:119
dccl::Exception
Exception class for DCCL.
Definition exception.h:47
dccl::Logger::is
bool is(logger::Verbosity verbosity, logger::Group group=logger::GENERAL)
Indicates the verbosity of the Logger until the next std::flush or std::endl. The boolean return is u...
Definition logger.h:191
dccl::Logger::connect
void connect(int verbosity_mask, Slot slot)
Connect the output of one or more given verbosities to a slot (function pointer or similar)
Definition logger.h:213
dccl::int32
google::protobuf::int32 int32
a signed 32 bit integer
Definition common.h:58
dccl::hex_encode
void hex_encode(CharIterator begin, CharIterator end, std::string *out, bool upper_case=false)
Encodes a (little-endian) hexadecimal string from a byte string. Index 0 of begin is written to index...
Definition binary.h:95