DCCL v5
Loading...
Searching...
No Matches
test.cpp
1// Copyright 2012-2023:
2// GobySoft, LLC (2013-)
3// Massachusetts Institute of Technology (2007-2014)
4// Community contributors (see AUTHORS file)
5// File authors:
6// Toby Schneider <toby@gobysoft.org>
7// Chris Murphy <cmurphy@aphysci.com>
8//
9//
10// This file is part of the Dynamic Compact Control Language Library
11// ("DCCL").
12//
13// DCCL is free software: you can redistribute it and/or modify
14// it under the terms of the GNU Lesser General Public License as published by
15// the Free Software Foundation, either version 2.1 of the License, or
16// (at your option) any later version.
17//
18// DCCL is distributed in the hope that it will be useful,
19// but WITHOUT ANY WARRANTY; without even the implied warranty of
20// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21// GNU Lesser General Public License for more details.
22//
23// You should have received a copy of the GNU Lesser General Public License
24// along with DCCL. If not, see <http://www.gnu.org/licenses/>.
25// tests arithmetic encoder
26
27#include "../../arithmetic/field_codec_arithmetic.h"
28#include "../../codec.h"
29
30#include "test_arithmetic.pb.h"
31
32#include "../../binary.h"
33using namespace dccl::test::arith;
34
35void run_test(dccl::arith::protobuf::ArithmeticModel& model,
36 const google::protobuf::Message& msg_in, bool set_model = true)
37{
38 static int i = 0;
39
40 static dccl::Codec codec;
41
42 if (!i)
43 {
44 void* dl_handle = dlopen(DCCL_ARITHMETIC_NAME, RTLD_LAZY);
45 if (!dl_handle)
46 {
47 std::cerr << "Failed to open " << DCCL_ARITHMETIC_NAME << std::endl;
48 exit(1);
49 }
50 codec.load_library(dl_handle);
51 }
52
53 if (set_model)
54 {
55 model.set_name("model");
56 dccl::arith::ModelManager::set_model(codec, model);
57 }
58
59 if (dccl::dlog.is(dccl::logger::INFO))
60 codec.info(msg_in.GetDescriptor(), &dccl::dlog);
61
62 codec.load(msg_in.GetDescriptor());
63
64 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "Message in:\n"
65 << msg_in.DebugString() << std::endl;
66
67 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "Try encode..." << std::endl;
68 std::string bytes;
69 codec.encode(&bytes, msg_in);
70 dccl::dlog.is(dccl::logger::INFO) &&
71 dccl::dlog << "... got bytes (hex): " << dccl::hex_encode(bytes) << std::endl;
72
73 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "Try decode..." << std::endl;
74
75 std::shared_ptr<google::protobuf::Message> msg_out(msg_in.New());
76 codec.decode(bytes, msg_out.get());
77
78 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "... got Message out:\n"
79 << msg_out->DebugString() << std::endl;
80
81 assert(msg_in.SerializeAsString() == msg_out->SerializeAsString());
82 ++i;
83}
84
85// usage: dccl_test10 [-v | 1]
86int main(int argc, char* argv[])
87{
88 bool verbose = false;
89 bool legacy_verbose = false;
90 for (int i = 1; i < argc; ++i)
91 {
92 if (argv[i] && argv[i][0] == '-' && argv[i][1] == 'v' && argv[i][2] == '\0')
93 verbose = true;
94 else if (argv[i] && std::string(argv[i]) == "1")
95 legacy_verbose = true;
96 }
97
98 if (verbose)
99 dccl::dlog.connect(dccl::logger::ALL, &std::cerr);
100 else if (legacy_verbose)
101 dccl::dlog.connect(dccl::logger::DEBUG3_PLUS, &std::cerr);
102 else
103 dccl::dlog.connect(dccl::logger::WARN_PLUS, &std::cerr);
104
105 dccl::Codec codec;
106
107 // test case from Practical Implementations of Arithmetic Coding by Paul G. Howard and Je rey Scott Vitter
108 {
109 dccl::arith::protobuf::ArithmeticModel model;
110
111 model.set_eof_frequency(4); // "a"
112
113 model.add_value_bound(0);
114 model.add_frequency(5); // "b"
115
116 model.add_value_bound(1);
117 model.add_frequency(1); // "EOF"
118
119 model.add_value_bound(2);
120
121 model.set_out_of_range_frequency(0);
122
123 ArithmeticDoubleTestMsg msg_in;
124
125 msg_in.add_value(0); // b
126 msg_in.add_value(0); // b
127 msg_in.add_value(0); // b
128 msg_in.add_value(1); // "EOF"
129
130 run_test(model, msg_in);
131 }
132
133 // misc test case
134 {
135 dccl::arith::protobuf::ArithmeticModel model;
136
137 model.add_value_bound(100.0);
138 model.add_frequency(100);
139
140 model.add_value_bound(100.1);
141 model.add_frequency(100);
142
143 model.add_value_bound(100.2);
144 model.add_frequency(100);
145
146 model.add_value_bound(100.3);
147 model.add_frequency(100);
148
149 model.add_value_bound(100.4);
150 model.add_frequency(90);
151
152 model.add_value_bound(100.5);
153 model.add_frequency(125);
154
155 model.add_value_bound(100.6);
156 model.add_frequency(125);
157
158 model.add_value_bound(100.7);
159 model.add_frequency(125);
160
161 model.add_value_bound(100.8);
162
163 model.set_eof_frequency(25);
164 model.set_out_of_range_frequency(10);
165
166 ArithmeticDoubleTestMsg msg_in;
167
168 msg_in.add_value(100.5);
169 msg_in.add_value(100.7);
170 msg_in.add_value(100.2);
171
172 run_test(model, msg_in);
173 }
174
175 // edge case 1, should be just a single bit ("1")
176 {
177 dccl::arith::protobuf::ArithmeticModel model;
178
179 model.set_eof_frequency(10);
180 model.set_out_of_range_frequency(0);
181
182 model.add_value_bound(1);
183 model.add_frequency(2);
184
185 model.add_value_bound(2);
186 model.add_frequency(3);
187
188 model.add_value_bound(3);
189 model.add_frequency(85);
190
191 model.add_value_bound(4);
192
193 ArithmeticEnumTestMsg msg_in;
194
195 msg_in.add_value(ENUM_C);
196 msg_in.add_value(ENUM_C);
197 msg_in.add_value(ENUM_C);
198 msg_in.add_value(ENUM_C);
199
200 run_test(model, msg_in);
201 }
202
203 // edge case 2, should be full 23 or 24 bits
204 {
205 dccl::arith::protobuf::ArithmeticModel model;
206
207 model.set_eof_frequency(10);
208 model.set_out_of_range_frequency(0);
209
210 model.add_value_bound(1);
211 model.add_frequency(2);
212
213 model.add_value_bound(2);
214 model.add_frequency(3);
215
216 model.add_value_bound(3);
217 model.add_frequency(85);
218
219 model.add_value_bound(4);
220
221 ArithmeticEnumTestMsg msg_in;
222
223 msg_in.add_value(ENUM_A);
224 msg_in.add_value(ENUM_A);
225 msg_in.add_value(ENUM_A);
226 msg_in.add_value(ENUM_A);
227
228 run_test(model, msg_in);
229 }
230
231 {
232 dccl::arith::protobuf::ArithmeticModel model;
233
234 model.set_eof_frequency(10);
235 model.set_out_of_range_frequency(0);
236
237 model.add_value_bound(1);
238 model.add_frequency(2);
239
240 model.add_value_bound(2);
241 model.add_frequency(3);
242
243 model.add_value_bound(3);
244 model.add_frequency(85);
245
246 model.add_value_bound(4);
247
248 ArithmeticSingleEnumTestMsg msg_in;
249
250 msg_in.set_value(ENUM_B);
251
252 run_test(model, msg_in);
253 }
254
255 // test case from Practical Implementations of Arithmetic Coding by Paul G. Howard and Je rey Scott Vitter
256 {
257 dccl::arith::protobuf::ArithmeticModel model;
258
259 model.set_eof_frequency(1);
260
261 model.add_value_bound(0);
262 model.add_frequency(1);
263
264 model.add_value_bound(1);
265 model.add_frequency(1);
266
267 model.add_value_bound(2);
268
269 model.set_out_of_range_frequency(1);
270
271 ArithmeticDouble3TestMsg msg_in;
272
273 msg_in.add_value(0);
274 msg_in.add_value(0);
275 msg_in.add_value(0);
276 msg_in.add_value(1);
277
278 model.set_is_adaptive(true);
279 run_test(model, msg_in);
280 run_test(model, msg_in, false);
281 run_test(model, msg_in, false);
282 run_test(model, msg_in, false);
283 }
284
285 // test case from Arithmetic Coding revealed: A guided tour from theory to praxis Sable Technical Report No. 2007-5 Eric Bodden
286
287 {
288 dccl::arith::protobuf::ArithmeticModel model;
289
290 model.set_eof_frequency(0);
291 model.set_out_of_range_frequency(0);
292
293 model.add_value_bound(1);
294 model.add_frequency(2);
295
296 model.add_value_bound(2);
297 model.add_frequency(1);
298
299 model.add_value_bound(3);
300 model.add_frequency(3);
301
302 model.add_value_bound(4);
303 model.add_frequency(1);
304
305 model.add_value_bound(5);
306 model.add_frequency(1);
307
308 model.add_value_bound(6);
309
310 ArithmeticEnum2TestMsg msg_in;
311
312 msg_in.add_value(ENUM2_A);
313 msg_in.add_value(ENUM2_B);
314 msg_in.add_value(ENUM2_C);
315 msg_in.add_value(ENUM2_C);
316 msg_in.add_value(ENUM2_E);
317 msg_in.add_value(ENUM2_D);
318 msg_in.add_value(ENUM2_A);
319 msg_in.add_value(ENUM2_C);
320
321 run_test(model, msg_in);
322 }
323
324 // randomly generate a model and a message
325 // loop over all message lengths from 0 to 100
326 srand(time(nullptr));
327 for (unsigned i = 0; i <= ArithmeticDouble2TestMsg::descriptor()
328 ->FindFieldByName("value")
329 ->options()
330 .GetExtension(dccl::field)
331 .max_repeat();
332 ++i)
333 {
334 dccl::arith::protobuf::ArithmeticModel model;
335
336 // pick some endpoints
337 dccl::int32 low = -(rand() % std::numeric_limits<dccl::int32>::max());
338 dccl::int32 high = rand() % std::numeric_limits<dccl::int32>::max();
339
340 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "low: " << low << ", high: " << high
341 << std::endl;
342
343 // number of symbols
344 dccl::int32 symbols = rand() % 1000 + 10;
345
346 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "symbols: " << symbols << std::endl;
347
348 // maximum freq
349 dccl::arith::Model::freq_type each_max_freq =
350 dccl::arith::Model::MAX_FREQUENCY / (symbols + 2);
351
352 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "each_max_freq: " << each_max_freq
353 << std::endl;
354
355 model.set_eof_frequency(rand() % each_max_freq + 1);
356 model.set_out_of_range_frequency(rand() % each_max_freq + 1);
357
358 model.add_value_bound(low);
359 model.add_frequency(rand() % each_max_freq + 1);
360 for (int j = 1; j < symbols; ++j)
361 {
362 // dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "j: " << j << std::endl;
363
364 dccl::int32 remaining_range = high - model.value_bound(j - 1);
365 model.add_value_bound(model.value_bound(j - 1) +
366 rand() % (remaining_range / symbols - j) + 1);
367 model.add_frequency(rand() % each_max_freq + 1);
368 }
369
370 model.add_value_bound(high);
371
372 ArithmeticDouble2TestMsg msg_in;
373
374 for (unsigned j = 0; j < i; ++j) msg_in.add_value(model.value_bound(rand() % symbols));
375
376 run_test(model, msg_in);
377
378 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "end random test #" << i << std::endl;
379 }
380
381 dccl::dlog.is(dccl::logger::INFO) && dccl::dlog << "all tests passed" << std::endl;
382}
dccl::Codec
The Dynamic CCL enCODer/DECoder. This is the main class you will use to load, encode and decode DCCL ...
Definition codec.h:61
dccl::Logger::is
bool is(logger::Verbosity verbosity, logger::Group group=logger::GENERAL)
Indicates the verbosity of the Logger until the next std::flush or std::endl. The boolean return is u...
Definition logger.h:191
dccl::Logger::connect
void connect(int verbosity_mask, Slot slot)
Connect the output of one or more given verbosities to a slot (function pointer or similar)
Definition logger.h:213
dccl::int32
google::protobuf::int32 int32
a signed 32 bit integer
Definition common.h:58
dccl::hex_encode
void hex_encode(CharIterator begin, CharIterator end, std::string *out, bool upper_case=false)
Encodes a (little-endian) hexadecimal string from a byte string. Index 0 of begin is written to index...
Definition binary.h:95