source: trunk/webif/pages_gen.c@ 8446

Last change on this file since 8446 was 8446, checked in by gf, 9 years ago

webif: Compress built-in templates.

Use the minilzo code that we already have to compress the built-in
templates. This saves ~90k binary size on 32bit defconfig. As the
tool reports:

Compressed 148820 template bytes into 51619 bytes. 97201 saved bytes (65.31%).

The old code that works with non-compressed templates is still here,
to use it, comment #define USE_COMPRESSION 1 in webif/pages_gen.c
and recompile.

Once we establish that template compression works fine the old
code can be removed.

This commit introduces no user visible changes.

File size: 14.5 KB
Line 
1/*
2 * OSCam WebIf pages generator
3 * Copyright (C) 2013 Unix Solutions Ltd.
4 *
5 * Authors: Georgi Chorbadzhiyski (gf@unixsol.org)
6 *
7 * This program is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20#include <ctype.h>
21#include <stdio.h>
22#include <stdarg.h>
23#include <stdlib.h>
24#include <stdbool.h>
25#include <string.h>
26#include <errno.h>
27#include <sys/types.h>
28#include <sys/stat.h>
29#include <unistd.h>
30#include <fcntl.h>
31#include <inttypes.h>
32#include <libgen.h>
33
34#define USE_COMPRESSION 1
35
36#include "../minilzo/minilzo.h"
37
38#define MAX_TEMPLATES 256
39static char *index_filename = "pages_index.txt";
40static char *output_pages_c = "pages.c";
41static char *output_pages_h = "pages.h";
42
43struct template {
44 char ident[64];
45 char file[128];
46 char deps[256];
47 uint32_t data_len;
48 enum { TXT, BIN } type;
49 uint8_t mime_type;
50#ifdef USE_COMPRESSION
51 uint8_t *buf;
52 size_t buf_len;
53 uint32_t ident_ofs;
54 uint32_t data_ofs;
55 uint32_t deps_ofs;
56#endif
57};
58
59struct templates {
60 unsigned int num;
61 struct template data[MAX_TEMPLATES];
62};
63
64static struct templates templates;
65static FILE *output_file;
66
67static void die(const char * s, ...) {
68 va_list args;
69 va_start(args, s);
70 fprintf(stderr, "ERROR: ");
71 vfprintf(stderr, s, args);
72 if (s[strlen(s) - 1] != '\n')
73 fprintf(stderr, "\n");
74 va_end(args);
75 exit(EXIT_FAILURE);
76}
77
78static FILE *xfopen(char *filename, char *mode) {
79 FILE *fh = fopen(filename, mode);
80 if (!fh)
81 die("fopen(%s, %s): %s\n", filename, mode, strerror(errno));
82 return fh;
83}
84
85static void readfile(const char *filename, uint8_t **data, size_t *data_len) {
86 struct stat sb;
87 if (stat(filename, &sb) != 0)
88 die("stat(%s): %s\n", filename, strerror(errno));
89 int fd = open(filename, O_RDONLY);
90 if (fd < 0)
91 die("open(%s): %s\n", filename, strerror(errno));
92 *data_len = sb.st_size;
93 *data = malloc(*data_len);
94 if (!*data)
95 die("%s(%s): can't alloc %zd bytes\n", __func__, filename, *data_len);
96 if (read(fd, *data, *data_len) < 0)
97 die("read(%d, %zd): %s\n", fd, *data_len, strerror(errno));
98}
99
100static bool is_text(char *filename) {
101 char *ext = strchr(basename(filename), '.');
102 if (ext) {
103 ext++;
104 if (strcmp(ext, "html") == 0) return true;
105 else if (strcmp(ext, "json") == 0) return true;
106 else if (strcmp(ext, "xml") == 0) return true;
107 else if (strcmp(ext, "css") == 0) return true;
108 else if (strcmp(ext, "svg") == 0) return true;
109 else if (strcmp(ext, "js") == 0) return true;
110 }
111 return false;
112}
113
114static uint8_t mime_type_from_filename(char *filename) {
115 char *ext = strchr(basename(filename), '.');
116 if (ext) {
117 ext++;
118 // See "enum template_types" bellow
119 if (strcmp(ext, "png") == 0) return 1;
120 else if (strcmp(ext, "gif") == 0) return 2;
121 else if (strcmp(ext, "ico") == 0) return 3;
122 else if (strcmp(ext, "jpg") == 0) return 4;
123 }
124 return 0;
125}
126
127static void parse_index_file(char *filename) {
128 FILE *f = xfopen(filename, "r");
129 int max_fields = 3;
130 char line[1024];
131 while (fgets(line, sizeof(line) - 1, f)) {
132 int field = 0, pos = 0;
133 char *ident = "", *file = "", *deps = "";
134 int len = strlen(line);
135 if (!len || !isalnum(line[0])) // Skip comments and junk
136 continue;
137 // Parse text[ ]text[ ]text
138 do {
139 while (line[pos] == ' ' || line[pos] == '\t') // Skip white space
140 pos++;
141 if (line[pos] == '\n')
142 break;
143 int start = pos;
144 while (line[pos] != ' ' && line[pos] != '\t' && line[pos] != '\n') // Data
145 pos++;
146 switch (++field) {
147 case 1: ident = line + start; line[pos] = '\0'; break;
148 case 2: file = line + start; line[pos] = '\0'; break;
149 case 3: deps = line + start; line[pos] = '\0'; break;
150 }
151 if (field >= max_fields)
152 break;
153 pos++;
154 } while (pos < len);
155 if (!strlen(ident) || !strlen(file))
156 continue;
157
158#define template_set(var) \
159 do { \
160 len = strlen(var); \
161 pos = sizeof(templates.data[0].var); \
162 if (len > pos - 1) \
163 die("%s=%s length exceeds maxlen (%d > %d)\n", #var, var, len, pos - 1); \
164 snprintf(templates.data[templates.num].var, pos, "%s", var); \
165 } while (0)
166 template_set(ident);
167 template_set(file);
168 template_set(deps);
169
170 templates.data[templates.num].type = is_text(file) ? TXT : BIN;
171 templates.data[templates.num].mime_type = mime_type_from_filename(file);
172 templates.num++;
173 if (templates.num == MAX_TEMPLATES - 1) {
174 die("Too many templates in %s. Maximum is %d. Increase MAX_TEMPLATES!\n",
175 filename, MAX_TEMPLATES);
176 }
177 }
178 fclose(f);
179}
180
181static void print_template(int tpl_idx) {
182 static bool ifdef_open = 0;
183 char *prev_deps = "";
184 char *next_deps = "";
185 char *ident = templates.data[tpl_idx].ident;
186 char *deps = templates.data[tpl_idx].deps;
187 if (tpl_idx > 0)
188 prev_deps = templates.data[tpl_idx - 1].deps;
189 if (tpl_idx + 1 < templates.num)
190 next_deps = templates.data[tpl_idx + 1].deps;
191 int deps_len = strlen(deps);
192
193 // Put guards
194 if (deps_len && strcmp(deps, prev_deps) != 0) {
195 int i, commas = 0;
196 for (i = 0; i < deps_len; i++) {
197 if (deps[i] == ',')
198 commas++;
199 }
200 if (commas == 0) {
201 fprintf(output_file, "#ifdef %s\n", deps);
202 } else {
203 char *ptr, *saveptr1 = NULL;
204 char *split_deps = strdup(deps);
205 for (i = 0, ptr = strtok_r(split_deps, ",", &saveptr1); ptr; ptr = strtok_r(NULL, ",", &saveptr1), i++) {
206 if (i == 0)
207 fprintf(output_file, "#if defined(%s)", ptr);
208 else
209 fprintf(output_file, " || defined(%s)", ptr);
210 }
211 fprintf(output_file, "\n");
212 free(split_deps);
213 }
214 ifdef_open = 1;
215 }
216
217#ifdef USE_COMPRESSION
218 fprintf(output_file, "\t{ .tpl_name_ofs=%5u, .tpl_data_ofs=%5u, .tpl_deps_ofs=%5u, .tpl_data_len=%5u, .tpl_type=%u }, /* %s %s %s */\n",
219 templates.data[tpl_idx].ident_ofs,
220 templates.data[tpl_idx].data_ofs,
221 templates.data[tpl_idx].deps_ofs,
222 templates.data[tpl_idx].data_len,
223 templates.data[tpl_idx].mime_type,
224 ident,
225 templates.data[tpl_idx].file,
226 deps
227 );
228#else
229 fprintf(output_file, "\t{ .tpl_name=\"%s\", .tpl_data=TPL%s, .tpl_deps=\"%s\", .tpl_data_len=%u, .tpl_type=%u },\n",
230 ident, ident, deps, templates.data[tpl_idx].data_len, templates.data[tpl_idx].mime_type
231 );
232#endif
233
234 if (ifdef_open && strcmp(deps, next_deps) != 0) {
235 fprintf(output_file, "#endif\n");
236 ifdef_open = 0;
237 }
238}
239
240#ifdef USE_COMPRESSION
241static void dump_cbinary(char *var_name, uint8_t *buf, size_t buf_len, size_t obuf_len) {
242 fprintf(output_file, "static const char *%s = \"", var_name);
243 int i;
244 for (i = 0; i < buf_len; i++) {
245 fprintf(output_file, "\\x%02x", buf[i]);
246 }
247 fprintf(output_file, "\";\n");
248 fprintf(output_file, "static const size_t %s_len = %zu;\n" , var_name, buf_len);
249 fprintf(output_file, "static const size_t %s_olen = %zu;\n\n", var_name, obuf_len);
250}
251
252#define HEAP_ALLOC(var, size) \
253 lzo_align_t __LZO_MMODEL var [ ((size) + (sizeof(lzo_align_t) - 1)) / sizeof(lzo_align_t) ]
254
255static HEAP_ALLOC(wrkmem, LZO1X_1_MEM_COMPRESS);
256#else
257
258static void dump_text(char *ident, uint8_t *buf, size_t buf_len) {
259 int i;
260 fprintf(output_file, "#define TPL%s \\\n\"", ident);
261 for (i = 0; i < buf_len; i++) {
262 switch (buf[i]) {
263 case '\n':
264 if (i < buf_len - 1)
265 fprintf(output_file, "\\n\\\n");
266 else
267 fprintf(output_file, "\\n");
268 break;
269 case '\\': fprintf(output_file, "\\\\"); break;
270 case '"' : fprintf(output_file, "\\\""); break;
271 default : fprintf(output_file, "%c", buf[i]); break;
272 }
273 }
274 fprintf(output_file, "\"\n\n");
275}
276
277static void dump_binary(char *ident, uint8_t *buf, size_t buf_len) {
278 fprintf(output_file, "#define TPL%s \\\n\"", ident);
279 int i;
280 for (i = 0; i < buf_len; i++) {
281 fprintf(output_file, "\\x%02x", buf[i]);
282 }
283 fprintf(output_file, "\"\n\n");
284}
285#endif
286
287int main(void) {
288 int i;
289
290 parse_index_file(index_filename);
291
292 output_file = xfopen(output_pages_h, "w");
293 fprintf(output_file, "#ifndef WEBIF_PAGES_H_\n");
294 fprintf(output_file, "#define WEBIF_PAGES_H_\n");
295 fprintf(output_file, "\n");
296 fprintf(output_file, "enum template_types {\n");
297 fprintf(output_file, " TEMPLATE_TYPE_TEXT = 0,\n");
298 fprintf(output_file, " TEMPLATE_TYPE_PNG = 1,\n");
299 fprintf(output_file, " TEMPLATE_TYPE_GIF = 2,\n");
300 fprintf(output_file, " TEMPLATE_TYPE_ICO = 3,\n");
301 fprintf(output_file, " TEMPLATE_TYPE_JPG = 4,\n");
302 fprintf(output_file, "};\n");
303 fprintf(output_file, "\n");
304#ifdef USE_COMPRESSION
305 fprintf(output_file, "#define COMPRESSED_TEMPLATES 1\n\n");
306 fprintf(output_file, "struct template {\n");
307 fprintf(output_file, " uint32_t tpl_name_ofs;\n");
308 fprintf(output_file, " uint32_t tpl_data_ofs;\n");
309 fprintf(output_file, " uint32_t tpl_deps_ofs;\n");
310 fprintf(output_file, " uint32_t tpl_data_len;\n");
311 fprintf(output_file, " uint8_t tpl_type;\n");
312 fprintf(output_file, "};\n");
313#else
314 fprintf(output_file, "struct template {\n");
315 fprintf(output_file, " char *tpl_name;\n");
316 fprintf(output_file, " char *tpl_data;\n");
317 fprintf(output_file, " char *tpl_deps;\n");
318 fprintf(output_file, " uint32_t tpl_data_len;\n");
319 fprintf(output_file, " uint8_t tpl_type;\n");
320 fprintf(output_file, "};\n");
321#endif
322 fprintf(output_file, "\n");
323 fprintf(output_file, "int32_t templates_count(void);\n");
324 fprintf(output_file, "bool template_is_image(enum template_types tpl_type);\n");
325 fprintf(output_file, "const char *template_get_mimetype(enum template_types tpl_type);\n");
326 fprintf(output_file, "const struct template *templates_get(void);\n");
327#ifdef USE_COMPRESSION
328 fprintf(output_file, "void templates_get_data(const char **data, size_t *data_len, size_t *odata_len);\n");
329#endif
330 fprintf(output_file, "\n");
331 fprintf(output_file, "#endif\n");
332 fclose(output_file);
333
334 output_file = xfopen(output_pages_c, "w");
335 fprintf(output_file, "#include \"../globals.h\"\n");
336 fprintf(output_file, "\n");
337 fprintf(output_file, "#ifdef WEBIF\n");
338 fprintf(output_file, "\n");
339 fprintf(output_file, "#include \"pages.h\"\n");
340 fprintf(output_file, "\n");
341
342#ifdef USE_COMPRESSION
343 // Calculate positions at which the values would be storred
344 uint32_t cur_pos = 0;
345 for (i = 0; i < templates.num; i++) {
346 struct template *t = &templates.data[i];
347 readfile(t->file, &t->buf, &t->buf_len);
348 t->data_len = t->buf_len;
349 // +1 to leave space for \0
350 t->ident_ofs = cur_pos; cur_pos += strlen(t->ident) + 1;
351 t->data_ofs = cur_pos; cur_pos += t->data_len + 1;
352 t->deps_ofs = cur_pos; cur_pos += strlen(t->deps) + 1;
353 }
354
355 // Allocate template data and populate it
356 #define data_len cur_pos
357 uint8_t *data = calloc(1, data_len);
358 if (!data)
359 die("Can't alloc %u bytes", data_len);
360 for (i = 0; i < templates.num; i++) {
361 struct template *t = &templates.data[i];
362 memcpy(data + t->ident_ofs, t->ident, strlen(t->ident));
363 memcpy(data + t->data_ofs , t->buf , t->buf_len);
364 free(t->buf);
365 if (!t->deps[0]) // No need to copy empty deps
366 continue;
367 memcpy(data + t->deps_ofs, t->deps, strlen(t->deps));
368 }
369 FILE *bin = xfopen("pages.bin", "w");
370 fwrite(data, data_len, 1, bin);
371 fclose(bin);
372
373 // Compress template data
374 lzo_uint in_len = data_len;
375 lzo_uint out_len = data_len + data_len / 16 + 64 + 3; // Leave enough space in the output
376 uint8_t *out = malloc(out_len);
377 if (!out)
378 die("Can't alloc %zu bytes", out_len);
379
380 if (lzo_init() != LZO_E_OK) {
381 fprintf(stderr, "internal error - lzo_init() failed !!!\n");
382 fprintf(stderr, "(this usually indicates a compiler bug - try recompiling\nwithout optimizations, and enable '-DLZO_DEBUG' for diagnostics)\n");
383 return 3;
384 }
385
386 int r = lzo1x_1_compress(data, in_len, out, &out_len, wrkmem);
387 if (r == LZO_E_OK) {
388 printf("GEN\tCompressed %lu template bytes into %lu bytes. %ld saved bytes (%.2f%%).\n",
389 (unsigned long)in_len, (unsigned long)out_len,
390 (long)in_len - (long)out_len, 100 - ((float)out_len / in_len) * 100);
391 } else {
392 /* this should NEVER happen */
393 printf("internal error - compression failed: %d\n", r);
394 return 2;
395 }
396
397 bin = xfopen("pages.bin.compressed", "w");
398 fwrite(out, out_len, 1, bin);
399 fclose(bin);
400
401 dump_cbinary("templates_data", out, out_len, data_len);
402 free(out);
403 free(data);
404#else
405 for (i = 0; i < templates.num; i++) {
406 uint8_t *buf;
407 size_t buf_len;
408 readfile(templates.data[i].file, &buf, &buf_len);
409 templates.data[i].data_len = buf_len;
410 switch (templates.data[i].type) {
411 case TXT: dump_text(templates.data[i].ident, buf, buf_len); break;
412 case BIN: dump_binary(templates.data[i].ident, buf, buf_len); break;
413 }
414 free(buf);
415 }
416#endif
417
418 fprintf(output_file, "static const struct template templates[] = {\n");
419 for (i = 0; i < templates.num; i++) {
420 print_template(i);
421 }
422 fprintf(output_file, "};\n");
423 fprintf(output_file, "\n");
424 fprintf(output_file, "int32_t templates_count(void) { return sizeof(templates) / sizeof(struct template); }\n");
425 fprintf(output_file, "const struct template *templates_get(void) { return templates; }\n");
426#ifdef USE_COMPRESSION
427 fprintf(output_file, "void templates_get_data(const char **data, size_t *data_len, size_t *data_olen) { *data = templates_data; *data_len = templates_data_len; *data_olen = templates_data_olen; }\n");
428#endif
429 fprintf(output_file, "\n");
430 fprintf(output_file, "bool template_is_image(enum template_types tpl_type) {\n");
431 fprintf(output_file, " switch (tpl_type) {\n");
432 fprintf(output_file, " case TEMPLATE_TYPE_PNG:\n");
433 fprintf(output_file, " case TEMPLATE_TYPE_GIF:\n");
434 fprintf(output_file, " case TEMPLATE_TYPE_ICO:\n");
435 fprintf(output_file, " case TEMPLATE_TYPE_JPG:\n");
436 fprintf(output_file, " return true;\n");
437 fprintf(output_file, " default:\n");
438 fprintf(output_file, " return false;\n");
439 fprintf(output_file, " }\n");
440 fprintf(output_file, " return false;\n");
441 fprintf(output_file, "}\n");
442 fprintf(output_file, "\n");
443 fprintf(output_file, "const char *template_get_mimetype(enum template_types tpl_type) {\n");
444 fprintf(output_file, " switch (tpl_type) {\n");
445 fprintf(output_file, " case TEMPLATE_TYPE_TEXT: return \"text/plain\";\n");
446 fprintf(output_file, " case TEMPLATE_TYPE_PNG : return \"image/png\";\n");
447 fprintf(output_file, " case TEMPLATE_TYPE_GIF : return \"image/gif\";\n");
448 fprintf(output_file, " case TEMPLATE_TYPE_ICO : return \"image/x-icon\";\n");
449 fprintf(output_file, " case TEMPLATE_TYPE_JPG : return \"image/jpg\";\n");
450 fprintf(output_file, " }\n");
451 fprintf(output_file, " return \"\";\n");
452 fprintf(output_file, "}\n");
453 fprintf(output_file, "\n");
454 fprintf(output_file, "#endif\n");
455 fclose(output_file);
456
457 return 0;
458}
Note: See TracBrowser for help on using the repository browser.