DCCL v3
test.cpp
1 // Copyright 2009-2017 Toby Schneider (http://gobysoft.org/index.wt/people/toby)
2 // GobySoft, LLC (for 2013-)
3 // Massachusetts Institute of Technology (for 2007-2014)
4 // Community contributors (see AUTHORS file)
5 //
6 //
7 // This file is part of the Dynamic Compact Control Language Library
8 // ("DCCL").
9 //
10 // DCCL is free software: you can redistribute it and/or modify
11 // it under the terms of the GNU Lesser General Public License as published by
12 // the Free Software Foundation, either version 2.1 of the License, or
13 // (at your option) any later version.
14 //
15 // DCCL is distributed in the hope that it will be useful,
16 // but WITHOUT ANY WARRANTY; without even the implied warranty of
17 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 // GNU Lesser General Public License for more details.
19 //
20 // You should have received a copy of the GNU Lesser General Public License
21 // along with DCCL. If not, see <http://www.gnu.org/licenses/>.
22 // tests arithmetic encoder
23 
24 #include <google/protobuf/descriptor.pb.h>
25 
26 #include "dccl/codec.h"
27 #include "dccl/arithmetic/field_codec_arithmetic.h"
28 
29 #include "test.pb.h"
30 
31 
32 #include "dccl/binary.h"
33 using namespace dccl::test;
34 
35 using dccl::operator<<;
36 
37 void run_test(dccl::arith::protobuf::ArithmeticModel& model,
38  const google::protobuf::Message& msg_in,
39  bool set_model = true)
40 {
41  static int i = 0;
42 
43  static dccl::Codec codec;
44 
45  if(!i)
46  {
47  void* dl_handle = dlopen(DCCL_ARITHMETIC_NAME, RTLD_LAZY);
48  if(!dl_handle)
49  {
50  std::cerr << "Failed to open " << DCCL_ARITHMETIC_NAME << std::endl;
51  exit(1);
52  }
53  codec.load_library(dl_handle);
54  }
55 
56  if(set_model)
57  {
58  model.set_name("model");
59  dccl::arith::ModelManager::set_model(model);
60  }
61 
62  codec.info(msg_in.GetDescriptor(), &std::cout);
63 
64  codec.load(msg_in.GetDescriptor());
65 
66  std::cout << "Message in:\n" << msg_in.DebugString() << std::endl;
67 
68 
69  std::cout << "Try encode..." << std::endl;
70  std::string bytes;
71  codec.encode(&bytes, msg_in);
72  std::cout << "... got bytes (hex): " << dccl::hex_encode(bytes) << std::endl;
73 
74  std::cout << "Try decode..." << std::endl;
75 
76  boost::shared_ptr<google::protobuf::Message> msg_out(msg_in.New());
77  codec.decode(bytes, msg_out.get());
78 
79  std::cout << "... got Message out:\n" << msg_out->DebugString() << std::endl;
80 
81  assert(msg_in.SerializeAsString() == msg_out->SerializeAsString());
82  ++i;
83 }
84 
85 
86 // usage: dccl_test10 [boolean: verbose]
87 int main(int argc, char* argv[])
88 {
89  if(argc > 1 && std::string(argv[1]) == "1")
90  dccl::dlog.connect(dccl::logger::DEBUG3_PLUS, &std::cerr);
91  else
92  dccl::dlog.connect(dccl::logger::DEBUG2_PLUS, &std::cerr);
93 
94  dccl::Codec codec;
95 
96  // test case from Practical Implementations of Arithmetic Coding by Paul G. Howard and Je rey Scott Vitter
97  {
99 
100  model.set_eof_frequency(4); // "a"
101 
102  model.add_value_bound(0);
103  model.add_frequency(5); // "b"
104 
105  model.add_value_bound(1);
106  model.add_frequency(1); // "EOF"
107 
108  model.add_value_bound(2);
109 
110  model.set_out_of_range_frequency(0);
111 
112  ArithmeticDoubleTestMsg msg_in;
113 
114  msg_in.add_value(0); // b
115  msg_in.add_value(0); // b
116  msg_in.add_value(0); // b
117  msg_in.add_value(1); // "EOF"
118 
119  run_test(model, msg_in);
120  }
121 
122  // misc test case
123  {
125 
126  model.add_value_bound(100.0);
127  model.add_frequency(100);
128 
129  model.add_value_bound(100.1);
130  model.add_frequency(100);
131 
132  model.add_value_bound(100.2);
133  model.add_frequency(100);
134 
135  model.add_value_bound(100.3);
136  model.add_frequency(100);
137 
138  model.add_value_bound(100.4);
139  model.add_frequency(90);
140 
141  model.add_value_bound(100.5);
142  model.add_frequency(125);
143 
144  model.add_value_bound(100.6);
145  model.add_frequency(125);
146 
147  model.add_value_bound(100.7);
148  model.add_frequency(125);
149 
150  model.add_value_bound(100.8);
151 
152  model.set_eof_frequency(25);
153  model.set_out_of_range_frequency(10);
154 
155  ArithmeticDoubleTestMsg msg_in;
156 
157  msg_in.add_value(100.5);
158  msg_in.add_value(100.7);
159  msg_in.add_value(100.2);
160 
161  run_test(model, msg_in);
162 
163  }
164 
165  // edge case 1, should be just a single bit ("1")
166  {
168 
169  model.set_eof_frequency(10);
170  model.set_out_of_range_frequency(0);
171 
172  model.add_value_bound(1);
173  model.add_frequency(2);
174 
175  model.add_value_bound(2);
176  model.add_frequency(3);
177 
178  model.add_value_bound(3);
179  model.add_frequency(85);
180 
181  model.add_value_bound(4);
182 
183  ArithmeticEnumTestMsg msg_in;
184 
185  msg_in.add_value(ENUM_C);
186  msg_in.add_value(ENUM_C);
187  msg_in.add_value(ENUM_C);
188  msg_in.add_value(ENUM_C);
189 
190  run_test(model, msg_in);
191  }
192 
193  // edge case 2, should be full 23 or 24 bits
194  {
196 
197  model.set_eof_frequency(10);
198  model.set_out_of_range_frequency(0);
199 
200  model.add_value_bound(1);
201  model.add_frequency(2);
202 
203  model.add_value_bound(2);
204  model.add_frequency(3);
205 
206  model.add_value_bound(3);
207  model.add_frequency(85);
208 
209  model.add_value_bound(4);
210 
211  ArithmeticEnumTestMsg msg_in;
212 
213  msg_in.add_value(ENUM_A);
214  msg_in.add_value(ENUM_A);
215  msg_in.add_value(ENUM_A);
216  msg_in.add_value(ENUM_A);
217 
218  run_test(model, msg_in);
219  }
220 
221 
222  {
224 
225  model.set_eof_frequency(10);
226  model.set_out_of_range_frequency(0);
227 
228  model.add_value_bound(1);
229  model.add_frequency(2);
230 
231  model.add_value_bound(2);
232  model.add_frequency(3);
233 
234  model.add_value_bound(3);
235  model.add_frequency(85);
236 
237  model.add_value_bound(4);
238 
239  ArithmeticSingleEnumTestMsg msg_in;
240 
241  msg_in.set_value(ENUM_B);
242 
243  run_test(model, msg_in);
244  }
245 
246  // test case from Practical Implementations of Arithmetic Coding by Paul G. Howard and Je rey Scott Vitter
247  {
249 
250  model.set_eof_frequency(1);
251 
252  model.add_value_bound(0);
253  model.add_frequency(1);
254 
255  model.add_value_bound(1);
256  model.add_frequency(1);
257 
258  model.add_value_bound(2);
259 
260  model.set_out_of_range_frequency(1);
261 
262  ArithmeticDouble3TestMsg msg_in;
263 
264  msg_in.add_value(0);
265  msg_in.add_value(0);
266  msg_in.add_value(0);
267  msg_in.add_value(1);
268 
269  model.set_is_adaptive(true);
270  run_test(model, msg_in);
271  run_test(model, msg_in, false);
272  run_test(model, msg_in, false);
273  run_test(model, msg_in, false);
274 
275  }
276 
277 
278  // test case from Arithmetic Coding revealed: A guided tour from theory to praxis Sable Technical Report No. 2007-5 Eric Bodden
279 
280  {
282 
283  model.set_eof_frequency(0);
284  model.set_out_of_range_frequency(0);
285 
286  model.add_value_bound(1);
287  model.add_frequency(2);
288 
289  model.add_value_bound(2);
290  model.add_frequency(1);
291 
292  model.add_value_bound(3);
293  model.add_frequency(3);
294 
295  model.add_value_bound(4);
296  model.add_frequency(1);
297 
298  model.add_value_bound(5);
299  model.add_frequency(1);
300 
301  model.add_value_bound(6);
302 
303  ArithmeticEnum2TestMsg msg_in;
304 
305  msg_in.add_value(ENUM2_A);
306  msg_in.add_value(ENUM2_B);
307  msg_in.add_value(ENUM2_C);
308  msg_in.add_value(ENUM2_C);
309  msg_in.add_value(ENUM2_E);
310  msg_in.add_value(ENUM2_D);
311  msg_in.add_value(ENUM2_A);
312  msg_in.add_value(ENUM2_C);
313 
314  run_test(model, msg_in);
315  }
316 
317  // randomly generate a model and a message
318  // loop over all message lengths from 0 to 100
319  srand ( time(NULL) );
320  for(unsigned i = 0; i <= ArithmeticDouble2TestMsg::descriptor()->FindFieldByName("value")->options().GetExtension(dccl::field).max_repeat(); ++i)
321  {
323 
324 
325  // pick some endpoints
326  dccl::int32 low = -(rand() % std::numeric_limits<dccl::int32>::max());
327  dccl::int32 high = rand() % std::numeric_limits<dccl::int32>::max();
328 
329  std::cout << "low: " << low << ", high: " << high << std::endl;
330 
331  // number of symbols
332  dccl::int32 symbols = rand() % 1000 + 10;
333 
334  std::cout << "symbols: " << symbols << std::endl;
335 
336  // maximum freq
337  dccl::arith::Model::freq_type each_max_freq = dccl::arith::Model::MAX_FREQUENCY / (symbols+2);
338 
339  std::cout << "each_max_freq: " << each_max_freq << std::endl;
340 
341  model.set_eof_frequency(rand() % each_max_freq + 1);
342  model.set_out_of_range_frequency(rand() % each_max_freq + 1);
343 
344  model.add_value_bound(low);
345  model.add_frequency(rand() % each_max_freq + 1);
346  for(int j = 1; j < symbols; ++j)
347  {
348 // std::cout << "j: " << j << std::endl;
349 
350  dccl::int32 remaining_range = high-model.value_bound(j-1);
351  model.add_value_bound(model.value_bound(j-1) + rand() % (remaining_range/symbols-j) +1);
352  model.add_frequency(rand() % each_max_freq + 1);
353  }
354 
355  model.add_value_bound(high);
356 
357  ArithmeticDouble2TestMsg msg_in;
358 
359  for(unsigned j = 0; j < i; ++j)
360  msg_in.add_value(model.value_bound(rand() % symbols));
361 
362 
363  run_test(model, msg_in);
364 
365  std::cout << "end random test #" << i << std::endl;
366 
367  }
368 
369 
370  std::cout << "all tests passed" << std::endl;
371 }
372 
The Dynamic CCL enCODer/DECoder. This is the main class you will use to load, encode and decode DCCL ...
Definition: codec.h:56
void info(std::ostream *os=0, int user_id=-1) const
Writes a human readable summary (including field sizes) of the provided DCCL type to the stream provi...
Definition: codec.h:155
void load()
All messages must be explicited loaded and validated (size checks, option extensions checks...
Definition: codec.h:96
void encode(std::string *bytes, const google::protobuf::Message &msg, bool header_only=false, int user_id=-1)
Encodes a DCCL message.
Definition: codec.cpp:273
void load_library(void *dl_handle)
Add codecs and/or load messages present in the given shared library handle.
Definition: codec.cpp:606
void connect(int verbosity_mask, Slot slot)
Connect the output of one or more given verbosities to a slot (function pointer or similar) ...
Definition: logger.h:161
void hex_encode(CharIterator begin, CharIterator end, std::string *out, bool upper_case=false)
Encodes a (little-endian) hexadecimal string from a byte string. Index 0 of begin is written to index...
Definition: binary.h:100
Unit test namespace.
Definition: test.cpp:44
CharIterator decode(CharIterator begin, CharIterator end, google::protobuf::Message *msg, bool header_only=false)
Decode a DCCL message when the type is known at compile time.
Definition: codec.h:439
google::protobuf::int32 int32
a signed 32 bit integer
Definition: common.h:57