Constant table compressor for integers in C++

Constablecom is a program that takes as input C++ source code containing integer constant tables, and converts those tables into C++ objects that provide the same interface as the original tables, but with (often) significantly less storage and slightly more execution time overhead. It is especially suited for compressing the tables generated by lexer/parsing tools such as flex, bison and yacc.

Constablecom reads stdin, writes stdout. Example use:

./constablecom < original.cpp > output.cpp

It uses a regexp to locate tables of integer numbers from the source code, and replaces them with classes providing identical external interface. The tables are required to be constant (not-written-to).

The pattern matched is: static $T $N?[?$D?]?=?{?$A?}?; Where ? matches optional whitespace, space matches mandatory whitespace, $T matches ([A-Za-z_0-9 ]*) (type), $N matches ([A-Za-z_][A-Za-z_0-9]*) (name), $D matches ([0-9]+) (dimension), and $A matches ([0-9, ]+) (array contents).

Restrictions:

Writing access will not work.
Calculating the table size using the sizeof(table)/sizeof(table[0]) idiom won't work.


 Iterating through the table using a pointer won't work.
 Creating references to table members won't work.
 Only read-only access through table[index] will work.



Constablecom has been written by Joel Yliluoma, a.k.a.
Bisqwit,

and is distributed under the terms of the
General Public License (GPL).



At least these GNU tools are required: gcc, make.

It also utilizes clamp (C++ lambda preprocessor), but it is included in the archive.

boost::regex and boost::smart_ptr are also required.



Example input:

static yyconst flex_int16_t yy_def[748] =
    {   0,
      733,    1,  733,    3,  734,  734,  735,  735,  736,  736,
      733,  733,  733,  733,  733,  737,  733,  733,  733,  733,
      733,  733,  733,  733,  733,  733,  733,  738,  738,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,  733,
      733,  733,  733,  733,  739,  733,  733,  740,  733,  733,
      741,  741,  733,  742,  733,  733,  733,  733,  733,  737,
      733,  733,  733,  733,  733,  733,  733,  733,  733,  733,
      733,  733,  733,  733,  733,  733,  733,  733,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,

       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,  743,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,  733,  733,  733,  744,  739,  733,  733,  733,
      733,  740,  741,  741,  733,  742,  733,  741,  745,  733,

      746,  733,  733,  733,  733,  733,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,  733,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,  747,   29,   29,   29,   29,  733,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,  733,   29,   29,   29,   29,   29,   29,   29,

       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,  733,  733,  733,  733,
      733,  733,  733,   29,   29,   29,   29,   29,   29,   29,
       29,   29,  733,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,  733,   29,   29,   29,   29,
      733,  733,   29,   29,   29,  733,   29,   29,   29,   29,
       29,  733,  747,   29,   29,  733,   29,   29,  733,   29,

       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,  733,  733,  733,  733,  733,
       29,  733,  733,  733,  733,  733,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,  733,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
      733,  733,  733,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,  733,   29,  733,   29,   29,
       29,   29,   29,  733,  733,   29,   29,  733,  733,  733,

      733,  733,   29,   29,   29,  733,   29,   29,   29,  733,
       29,   29,   29,   29,   29,   29,   29,  733,  733,   29,
      733,   29,   29,  733,  733,  733,  733,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,   29,
       29,   29,   29,   29,   29,   29,   29,   29,   29,  733,
       29,   29,   29,   29,   29,   29,  733,   29,   29,  733,
      733,  733,   29,   29,   29,   29,   29,   29,   29,  733,
       29,   29,   29,   29,   29,  733,  733,   29,   29,  733,
      733,  733,  733,  733,  733,   29,  733,  733,  733,   29,
       29,   29,  733,  733,  733,   29,   29,  733,  733,   29,

      733,  733,   29,  733,  733,  733,  733,  733,   29,   29,
       29,   29,  733,   29,   29,   29,   29,  733,  733,   29,
       29,   29,   29,  733,   29,   29,   29,   29,  733,   29,
       29,  733,  733,  733,  733,   29,  733,  733,  733,  733,
      733,  733,  733,  733,   29,  733,  733,  733,  733,  733,
       29,   29,   29,  733,   29,  733,  733,  733,  733,  733,
      733,  733,   29,  733,   29,  733,  733,  733,  733,  733,
      733,  733,  733,  733,  733,  733,  733,  733,  733,  733,
      733,  733,   29,   29,   29,  733,  733,  733,  733,  733,
      733,  733,  733,  733,  733,  733,  733,  733,  733,  733,

      733,  733,  733,  733,  733,  733,  733,  733,  733,  733,
      733,  733,  733,  733,  733,  733,  733,  733,  733,  733,
      733,  733,  733,  733,  733,  733,  733,  733,  733,  733,
      733,  733,    0,  733,  733,  733,  733,  733,  733,  733,
      733,  733,  733,  733,  733,  733,  733
    } ;
/* Amount of data: 2*748 = 1496 bytes */

Example output corresponding that input (indentation added manually):

#define __STDC_CONSTANT_MACROS
#include <stdint.h>
static struct yy_def_generator {
  yyconst flex_int16_t operator[] (unsigned index) const;
} yy_def;
yyconst flex_int16_t yy_def_generator::operator[] (unsigned index) const {
  if (index < 17) {
    static const uint_least16_t datatable[17] =
      { 0,733,1,733,3,734,734,735,735,736,736,733,733,733,733,
      733,737,
    };
    return datatable[index - 0] + 0;
  }
  if (index >= 28 && index < 30) {
    static const uint_least8_t datatable[2] = { 0,0,};
    return datatable[index - 28] + 738;
  }
  if (index >= 55 && index < 71) {
    static const uint_least8_t datatable[16] =
      { 6,0,0,7,0,0,8,8,0,9,0,0,0,0,0,4,};
    return datatable[index - 55] + 733;
  }
  if (index == 118) return 743;
  if (index >= 186 && index < 202) {
    static const uint_least8_t datatable[16] =
      { 11,6,0,0,0,0,7,8,8,0,9,0,8,12,0,13,};
    return datatable[index - 186] + 733;
  }
  if (index == 254) return 747;
  if (index == 393) return 747;
  if (index == 733) return 0;
  if (index >= 30 && index < 686) {
  /* 463 matching out of 656, 35 dontcare */
    static const uint_least32_t bitset[21] = {
      UINT32_C(4293918720), UINT32_C(134217727), UINT32_C(16777216),
      UINT32_C(0), UINT32_C(4261412864), UINT32_C(131071),
      UINT32_C(262144), UINT32_C(33), UINT32_C(128), UINT32_C(3758096384),
      UINT32_C(2214600719), UINT32_C(150545), UINT32_C(8060),
      UINT32_C(234881536), UINT32_C(301139200), UINT32_C(248577),
      UINT32_C(270303488), UINT32_C(3677941708), UINT32_C(3162779783),
      UINT32_C(4127006591), UINT32_C(8191)
    };
    unsigned bitslot = (index - 30) / 32;
    unsigned bitpos = (index - 30) % 32;
    if (!(bitset[bitslot] & (UINT32_C(1) << bitpos)))
      return 29;
  }
  if (index >= 1 && index < 89) {
  /* 51 matching out of 88, 54 dontcare */
    return 733;
  }
  if (index >= 183) {
  /* 203 matching out of 565, 369 dontcare */
    return 733;
  }
  return 0;
}
/* Amount of data: 2*17 + 2 + 16 + 16 + 4*21 = 152 bytes, plus some code */


   The official home page of constablecom
   is at http://iki.fi/bisqwit/source/constablecom.html.

   Check there for new versions.

Constant table compressor for integers in C++

0. Contents

1. Purpose

2. Usage

`3. Copying`

`4. Requirements`

`5. Example`

`6. Downloading`