Changeset 8446


Ignore:
Timestamp:
03/01/13 11:56:51 (8 years ago)
Author:
gf
Message:

webif: Compress built-in templates.

Use the minilzo code that we already have to compress the built-in
templates. This saves ~90k binary size on 32bit defconfig. As the
tool reports:

Compressed 148820 template bytes into 51619 bytes. 97201 saved bytes (65.31%).

The old code that works with non-compressed templates is still here,
to use it, comment #define USE_COMPRESSION 1 in webif/pages_gen.c
and recompile.

Once we establish that template compression works fine the old
code can be removed.

This commit introduces no user visible changes.

Location:
trunk
Files:
5 edited

Legend:

Unmodified
Added
Removed
  • trunk/config.sh

    r8395 r8446  
    309309    have_flag USE_PCSC && echo "CONFIG_CARDREADER_PCSC=y" || echo "# CONFIG_CARDREADER_PCSC=n"
    310310    # Extra modules/libraries
    311     enabled MODULE_GBOX && echo "CONFIG_LIB_MINILZO=y" || echo "# CONFIG_LIB_MINILZO=n"
     311    enabled_any MODULE_GBOX WEBIF && echo "CONFIG_LIB_MINILZO=y" || echo "# CONFIG_LIB_MINILZO=n"
    312312    not_have_flag USE_LIBCRYPTO && echo "CONFIG_LIB_AES=y" || echo "# CONFIG_LIB_AES=n"
    313313    enabled MODULE_CCCAM && echo "CONFIG_LIB_RC6=y" || echo "# CONFIG_LIB_RC6=n"
  • trunk/module-webif-tpl.c

    r8445 r8446  
    66#include "oscam-files.h"
    77#include "oscam-string.h"
     8#ifdef COMPRESSED_TEMPLATES
     9#include "minilzo/minilzo.h"
     10#endif
    811
    912extern uint8_t cs_http_use_utf8;
     
    2427
    2528static struct tpl *tpls;
     29static char *tpls_data;
    2630static int tpls_count;
    2731
     
    4549void webif_tpls_prepare(void) {
    4650    int i;
    47     extern const struct template templates[];
     51    const struct template *templates = templates_get();
    4852    tpls_count = templates_count();
    4953    if (!cs_malloc(&tpls, tpls_count * sizeof(struct tpl))) {
     
    5155        return;
    5256    }
     57#ifdef COMPRESSED_TEMPLATES
     58    const char *templates_cdata;
     59    size_t tpls_data_len, tpls_data_olen;
     60    templates_get_data(&templates_cdata, &tpls_data_len, &tpls_data_olen);
     61    if (!cs_malloc(&tpls_data, tpls_data_olen)) {
     62        tpls_count = 0;
     63        return;
     64    }
     65
     66    lzo_uint new_len = tpls_data_olen;
     67    int r = lzo1x_decompress_safe((uint8_t *)templates_cdata, tpls_data_len, (uint8_t *)tpls_data, &new_len, NULL);
     68    if (r == LZO_E_OK && new_len == tpls_data_olen) {
     69        cs_log("webif: decompressed %zu bytes back into %zu bytes", tpls_data_len, tpls_data_olen);
     70    } else {
     71        /* this should NEVER happen */
     72        cs_log("internal error - decompression failed: %d\n", r);
     73        free(tpls);
     74        tpls_count = 0;
     75    }
     76
     77    for(i = 0; i < tpls_count; ++i) {
     78        tpls[i].tpl_name      = tpls_data + templates[i].tpl_name_ofs;
     79        tpls[i].tpl_data      = tpls_data + templates[i].tpl_data_ofs;
     80        tpls[i].tpl_deps      = tpls_data + templates[i].tpl_deps_ofs;
     81        tpls[i].tpl_data_len  = templates[i].tpl_data_len;
     82        tpls[i].tpl_type      = templates[i].tpl_type;
     83        tpls[i].tpl_name_hash = jhash(tpls[i].tpl_name, strlen(tpls[i].tpl_name));
     84        tpl_init_base64(&tpls[i]);
     85    }
     86#else
    5387    for(i = 0; i < tpls_count; ++i) {
    5488        tpls[i].tpl_name_hash = jhash(templates[i].tpl_name, strlen(templates[i].tpl_name));
     
    6094        tpl_init_base64(&tpls[i]);
    6195    }
     96#endif
    6297}
    6398
     
    67102        free(tpls[i].extra_data);
    68103    }
     104    free(tpls_data);
    69105    free(tpls);
    70106}
  • trunk/webif/.gitignore

    r8418 r8446  
    11pages.c
    22pages.dep
     3pages.bin
     4pages.bin.compressed
    35pages.h
    46pages_gen
  • trunk/webif/Makefile

    r8429 r8446  
    2424pages_gen: Makefile pages_gen.c
    2525    $(SAY) "HOSTCC  webif/$@"
    26     $(Q)$(HOSTCC) $(HOSTCFLAGS) pages_gen.c -o $@
     26    $(Q)$(HOSTCC) $(HOSTCFLAGS) ../minilzo/minilzo.c pages_gen.c -o $@
    2727
    2828clean:
    29     @-for FILE in pages_gen pages.dep pages.h pages.c; do \
     29    @-for FILE in pages_gen pages.dep pages.bin pages.bin.compressed pages.h pages.c; do \
    3030        if [ -f $$FILE ]; then echo "RM webif/$$FILE"; fi; \
    3131        rm -rf $$FILE; \
  • trunk/webif/pages_gen.c

    r8445 r8446  
    3232#include <libgen.h>
    3333
     34#define USE_COMPRESSION 1
     35
     36#include "../minilzo/minilzo.h"
     37
    3438#define MAX_TEMPLATES 256
    3539static char *index_filename = "pages_index.txt";
     
    3741static char *output_pages_h = "pages.h";
    3842
     43struct template {
     44    char ident[64];
     45    char file[128];
     46    char deps[256];
     47    uint32_t data_len;
     48    enum { TXT, BIN } type;
     49    uint8_t mime_type;
     50#ifdef USE_COMPRESSION
     51    uint8_t *buf;
     52    size_t buf_len;
     53    uint32_t ident_ofs;
     54    uint32_t data_ofs;
     55    uint32_t deps_ofs;
     56#endif
     57};
     58
    3959struct templates {
    4060    unsigned int num;
    41     struct {
    42         char ident[64];
    43         char file[128];
    44         char deps[256];
    45         uint32_t data_len;
    46         enum { TXT, BIN } type;
    47         uint8_t mime_type;
    48     } data[MAX_TEMPLATES];
     61    struct template data[MAX_TEMPLATES];
    4962};
    5063
     
    202215    }
    203216
     217#ifdef USE_COMPRESSION
     218    fprintf(output_file, "\t{ .tpl_name_ofs=%5u, .tpl_data_ofs=%5u, .tpl_deps_ofs=%5u, .tpl_data_len=%5u, .tpl_type=%u }, /* %s %s %s */\n",
     219        templates.data[tpl_idx].ident_ofs,
     220        templates.data[tpl_idx].data_ofs,
     221        templates.data[tpl_idx].deps_ofs,
     222        templates.data[tpl_idx].data_len,
     223        templates.data[tpl_idx].mime_type,
     224        ident,
     225        templates.data[tpl_idx].file,
     226        deps
     227    );
     228#else
    204229    fprintf(output_file, "\t{ .tpl_name=\"%s\", .tpl_data=TPL%s, .tpl_deps=\"%s\", .tpl_data_len=%u, .tpl_type=%u },\n",
    205230        ident, ident, deps, templates.data[tpl_idx].data_len, templates.data[tpl_idx].mime_type
    206231    );
     232#endif
    207233
    208234    if (ifdef_open && strcmp(deps, next_deps) != 0) {
     
    211237    }
    212238}
     239
     240#ifdef USE_COMPRESSION
     241static void dump_cbinary(char *var_name, uint8_t *buf, size_t buf_len, size_t obuf_len) {
     242    fprintf(output_file, "static const char   *%s     = \"", var_name);
     243    int i;
     244    for (i = 0; i < buf_len; i++) {
     245        fprintf(output_file, "\\x%02x", buf[i]);
     246    }
     247    fprintf(output_file, "\";\n");
     248    fprintf(output_file, "static const size_t %s_len  = %zu;\n"  , var_name, buf_len);
     249    fprintf(output_file, "static const size_t %s_olen = %zu;\n\n", var_name, obuf_len);
     250}
     251
     252#define HEAP_ALLOC(var, size) \
     253    lzo_align_t __LZO_MMODEL var [ ((size) + (sizeof(lzo_align_t) - 1)) / sizeof(lzo_align_t) ]
     254
     255static HEAP_ALLOC(wrkmem, LZO1X_1_MEM_COMPRESS);
     256#else
    213257
    214258static void dump_text(char *ident, uint8_t *buf, size_t buf_len) {
     
    239283    fprintf(output_file, "\"\n\n");
    240284}
     285#endif
    241286
    242287int main(void) {
     
    257302    fprintf(output_file, "};\n");
    258303    fprintf(output_file, "\n");
     304#ifdef USE_COMPRESSION
     305    fprintf(output_file, "#define COMPRESSED_TEMPLATES 1\n\n");
     306    fprintf(output_file, "struct template {\n");
     307    fprintf(output_file, "  uint32_t tpl_name_ofs;\n");
     308    fprintf(output_file, "  uint32_t tpl_data_ofs;\n");
     309    fprintf(output_file, "  uint32_t tpl_deps_ofs;\n");
     310    fprintf(output_file, "  uint32_t tpl_data_len;\n");
     311    fprintf(output_file, "  uint8_t tpl_type;\n");
     312    fprintf(output_file, "};\n");
     313#else
    259314    fprintf(output_file, "struct template {\n");
    260315    fprintf(output_file, "  char *tpl_name;\n");
     
    264319    fprintf(output_file, "  uint8_t tpl_type;\n");
    265320    fprintf(output_file, "};\n");
     321#endif
    266322    fprintf(output_file, "\n");
    267323    fprintf(output_file, "int32_t templates_count(void);\n");
    268324    fprintf(output_file, "bool template_is_image(enum template_types tpl_type);\n");
    269325    fprintf(output_file, "const char *template_get_mimetype(enum template_types tpl_type);\n");
     326    fprintf(output_file, "const struct template *templates_get(void);\n");
     327#ifdef USE_COMPRESSION
     328    fprintf(output_file, "void templates_get_data(const char **data, size_t *data_len, size_t *odata_len);\n");
     329#endif
    270330    fprintf(output_file, "\n");
    271331    fprintf(output_file, "#endif\n");
     
    280340    fprintf(output_file, "\n");
    281341
     342#ifdef USE_COMPRESSION
     343    // Calculate positions at which the values would be storred
     344    uint32_t cur_pos = 0;
     345    for (i = 0; i < templates.num; i++) {
     346        struct template *t = &templates.data[i];
     347        readfile(t->file, &t->buf, &t->buf_len);
     348        t->data_len = t->buf_len;
     349        // +1 to leave space for \0
     350        t->ident_ofs = cur_pos; cur_pos += strlen(t->ident) + 1;
     351        t->data_ofs  = cur_pos; cur_pos += t->data_len      + 1;
     352        t->deps_ofs  = cur_pos; cur_pos += strlen(t->deps)  + 1;
     353    }
     354
     355    // Allocate template data and populate it
     356    #define data_len cur_pos
     357    uint8_t *data = calloc(1, data_len);
     358    if (!data)
     359        die("Can't alloc %u bytes", data_len);
     360    for (i = 0; i < templates.num; i++) {
     361        struct template *t = &templates.data[i];
     362        memcpy(data + t->ident_ofs, t->ident, strlen(t->ident));
     363        memcpy(data + t->data_ofs , t->buf  , t->buf_len);
     364        free(t->buf);
     365        if (!t->deps[0]) // No need to copy empty deps
     366            continue;
     367        memcpy(data + t->deps_ofs, t->deps, strlen(t->deps));
     368    }
     369    FILE *bin = xfopen("pages.bin", "w");
     370    fwrite(data, data_len, 1, bin);
     371    fclose(bin);
     372
     373    // Compress template data
     374    lzo_uint in_len  = data_len;
     375    lzo_uint out_len = data_len + data_len / 16 + 64 + 3; // Leave enough space in the output
     376    uint8_t *out = malloc(out_len);
     377    if (!out)
     378        die("Can't alloc %zu bytes", out_len);
     379
     380    if (lzo_init() != LZO_E_OK) {
     381        fprintf(stderr, "internal error - lzo_init() failed !!!\n");
     382        fprintf(stderr, "(this usually indicates a compiler bug - try recompiling\nwithout optimizations, and enable '-DLZO_DEBUG' for diagnostics)\n");
     383        return 3;
     384    }
     385
     386    int r = lzo1x_1_compress(data, in_len, out, &out_len, wrkmem);
     387    if (r == LZO_E_OK) {
     388        printf("GEN\tCompressed %lu template bytes into %lu bytes. %ld saved bytes (%.2f%%).\n",
     389            (unsigned long)in_len, (unsigned long)out_len,
     390            (long)in_len - (long)out_len, 100 - ((float)out_len / in_len) * 100);
     391    } else {
     392        /* this should NEVER happen */
     393        printf("internal error - compression failed: %d\n", r);
     394        return 2;
     395    }
     396
     397    bin = xfopen("pages.bin.compressed", "w");
     398    fwrite(out, out_len, 1, bin);
     399    fclose(bin);
     400
     401    dump_cbinary("templates_data", out, out_len, data_len);
     402    free(out);
     403    free(data);
     404#else
    282405    for (i = 0; i < templates.num; i++) {
    283406        uint8_t *buf;
     
    291414        free(buf);
    292415    }
    293 
    294     fprintf(output_file, "const struct template templates[] = {\n");
     416#endif
     417
     418    fprintf(output_file, "static const struct template templates[] = {\n");
    295419    for (i = 0; i < templates.num; i++) {
    296420        print_template(i);
     
    299423    fprintf(output_file, "\n");
    300424    fprintf(output_file, "int32_t templates_count(void) { return sizeof(templates) / sizeof(struct template); }\n");
     425    fprintf(output_file, "const struct template *templates_get(void) { return templates; }\n");
     426#ifdef USE_COMPRESSION
     427    fprintf(output_file, "void templates_get_data(const char **data, size_t *data_len, size_t *data_olen) { *data = templates_data; *data_len = templates_data_len; *data_olen = templates_data_olen; }\n");
     428#endif
    301429    fprintf(output_file, "\n");
    302430    fprintf(output_file, "bool template_is_image(enum template_types tpl_type) {\n");
Note: See TracChangeset for help on using the changeset viewer.