Mercurial > hg > dmlib
view tools/fanalyze.c @ 2258:c146033f1f6a
Cleanups.
author | Matti Hamalainen <ccr@tnsp.org> |
---|---|
date | Sun, 16 Jun 2019 23:31:58 +0300 |
parents | 6ba4aed198e4 |
children | 3739e2ac8bb1 |
line wrap: on
line source
/* * Fanalyze - Commandline tool for analyzing similarities between multiple files * Programmed and designed by Matti 'ccr' Hamalainen * (C) Copyright 2018-2019 Tecnic Software productions (TNSP) * * Please read file 'COPYING' for information on license and distribution. */ #include "dmtool.h" #include "dmlib.h" #include "dmargs.h" #include "dmfile.h" #define SET_MAX_FILES 64 #define SET_MAX_ELEMS 256 #define SET_MAX_VALUES 64 #define SET_MAX_GREPLIST 64 /* Typedefs */ typedef struct { Uint8 stats[SET_MAX_ELEMS]; Uint8 variants, data; int interest[16]; int interestF[16]; } DMCompElem; typedef struct { int count; Uint8 value; } DMStatValue; typedef struct { DMStatValue cv[SET_MAX_ELEMS]; } DMStats; typedef struct { char *filename; Uint8 *data; size_t size; DMStats stats; } DMSourceFile; enum { DMGV_UINT8 = 0, DMGV_UINT16_LE, DMGV_UINT16_BE, DMGV_UINT32_LE, DMGV_UINT32_BE, DMGV_last }; enum { DMGS_HEX = 0, DMGS_DEC, DMGS_last }; typedef struct { int type; int disp; int nvalues; Uint32 values[SET_MAX_GREPLIST]; BOOL vwildcards[SET_MAX_GREPLIST]; } DMGrepValue; typedef struct { char *name; Uint32 nmax; unsigned int bsize; } DMGrepType; static const DMGrepType dmGrepTypes[DMGV_last] = { { "8bit (byte)" , (1ULL << 8) - 1, 1 }, { "16bit (word) LE" , (1ULL << 16) - 1, 2 }, { "16bit (word) BE" , (1ULL << 16) - 1, 2 }, { "32bit (word) LE" , (1ULL << 32) - 1, 4 }, { "32bit (word) BE" , (1ULL << 32) - 1, 4 }, }; typedef struct { char *name; char *fmtPrefix; char *fmt; } DMGrepDisp; static const DMGrepDisp dmGrepDisp[DMGS_last] = { { "hex", "0", "x" }, { "dec", "" , "d" }, }; enum { FA_ANALYZE, FA_GREP, FA_OFFSET, }; /* Global variables */ int setMode = FA_ANALYZE; int nsrcFiles = 0; // Number of source files DMSourceFile srcFiles[SET_MAX_FILES]; // Source file names DMStats totalStats; int nsetGrepValues = 0; DMGrepValue setGrepValues[SET_MAX_VALUES]; /* Arguments */ static const DMOptArg optList[] = { { 0, '?', "help", "Show this help", OPT_NONE }, { 1, 'v', "verbose", "Be more verbose", OPT_NONE }, { 2, 'g', "grep", "Binary grep <val>[,<val2>...][:<le|be>[8|16|32]]", OPT_ARGREQ }, { 3, 'o', "offset", "Show data in offset <offs>[,<offs2>...][:<le|be>[8|16|32][d|x]]", OPT_ARGREQ }, }; static const int optListN = sizeof(optList) / sizeof(optList[0]); void argShowHelp() { dmPrintBanner(stdout, dmProgName, "[options] <input file #1> <input file #2> [...]"); dmArgsPrintHelp(stdout, optList, optListN, 0); fprintf(stdout, "\n" "Fanalyze is a simplistic commandline tool to assist analysis of similarities\n" "between multiple files of same format (but different content). It provides\n" "automatic analysis (default operating mode), binary grep functionality (-g)\n" "and offset data display (-o)\n" "\n" "Value lists for grep function can contain wildcard '?' (or '#') which\n" "matches any value of the specified (or inferred) type. For example:\n" "-g 0x0f,7,5,?,5,?,? will match sequence of bytes 0f 07 05 ?? 05 ?? ??\n" "and -g 0xe,0x1001,?,2023:le16 will match le16 value 000e 1001 ???? 07e7\n" ); } BOOL dmGetData(const int type, const DMSourceFile *file, const size_t offs, Uint32 *mval) { Uint8 *data = file->data + offs; if (offs + dmGrepTypes[type].bsize >= file->size) { *mval = 0; return FALSE; } switch (type) { case DMGV_UINT8: *mval = *((Uint8 *) data); break; case DMGV_UINT16_LE: *mval = DM_LE16_TO_NATIVE(*((Uint16 *) data)); break; case DMGV_UINT16_BE: *mval = DM_BE16_TO_NATIVE(*((Uint16 *) data)); break; case DMGV_UINT32_LE: *mval = DM_LE32_TO_NATIVE(*((Uint32 *) data)); break; case DMGV_UINT32_BE: *mval = DM_BE32_TO_NATIVE(*((Uint32 *) data)); break; default: *mval = 0; return FALSE; } return TRUE; } void dmPrintGrepValueList(const DMGrepValue *node, const BOOL match, DMSourceFile *file, const size_t offs) { char mfmt[16]; snprintf(mfmt, sizeof(mfmt), "%%%s%d%s%%s", dmGrepDisp[node->disp].fmtPrefix, dmGrepTypes[node->type].bsize * 2, dmGrepDisp[node->disp].fmt); for (int n = 0; n < node->nvalues; n++) { const char *veol = (n + 1 < node->nvalues) ? " " : "\n"; if (match) { Uint32 mval; dmGetData(node->type, file, offs + n, &mval); dmPrint(1, mfmt, mval, veol); } else { if (node->vwildcards[n]) dmPrint(1, "?%s", veol); else dmPrint(1, mfmt, node->values[n], veol); } } } int argParseGrepValue(const char *arg, const int mode) { const char *specsep = strchr(arg, ':'); char *vspec, *vstr, *vsep; DMGrepValue val; int ret = DMERR_OK; BOOL more; memset(&val, 0, sizeof(val)); if (setMode != FA_ANALYZE && setMode != mode) { dmErrorMsg("Options specifying multiple operating modes can't be used.\n"); return DMERR_INVALID_ARGS; } setMode = mode; // Do we have spec? if (specsep != NULL) { vspec = dm_strdup_trim(specsep + 1, DM_TRIM_BOTH); vstr = dm_strndup_trim(arg, specsep - arg, DM_TRIM_BOTH); } else { vspec = NULL; vstr = dm_strdup(arg); } // Parse spec if any if (vspec != NULL) { BOOL vendianess = TRUE; char *vtmp = vspec; // Get endianess specifier, if any if (dm_strncasecmp(vtmp, "le", 2) == 0) { vendianess = TRUE; vtmp += 2; } else if (dm_strncasecmp(vtmp, "be", 2) == 0) { vendianess = FALSE; vtmp += 2; } // Get value bit size if (strncmp(vtmp, "8", 1) == 0) { val.type = DMGV_UINT8; vtmp += 1; } else if (strncmp(vtmp, "16", 2) == 0) { val.type = vendianess ? DMGV_UINT16_LE : DMGV_UINT16_BE; vtmp += 2; } else if (strncmp(vtmp, "32", 2) == 0) { val.type = vendianess ? DMGV_UINT32_LE : DMGV_UINT32_BE; vtmp += 2; } else { ret = dmError(DMERR_INVALID_ARGS, "Invalid grep type '%s'.\n", vspec); goto out; } switch (tolower(*vtmp)) { case 'd': val.disp = DMGS_DEC; break; case 'x': case 'h': val.disp = DMGS_HEX; break; case 0: break; default: ret = dmError(DMERR_INVALID_ARGS, "Invalid grep view type '%s'.\n", vspec); goto out; } } // Get list of values char *vtmp = vstr; do { if (val.nvalues >= SET_MAX_GREPLIST) { ret = dmError(DMERR_BOUNDS, "Too many greplist values specified '%s'.\n", vstr); goto out; } if ((vsep = strchr(vtmp, ',')) != NULL) { *vsep = 0; more = TRUE; } else more = FALSE; if (vtmp[0] == '#' || vtmp[0] == '?') { val.vwildcards[val.nvalues] = TRUE; if (mode == FA_OFFSET) { ret = dmError(DMERR_INVALID_ARGS, "Offset mode does not allow wildcard values.\n"); goto out; } } else if (!dmGetIntVal(vtmp, &val.values[val.nvalues], NULL)) { ret = dmError(DMERR_INVALID_ARGS, "Not a valid integer value '%s'.\n", vtmp); goto out; } val.nvalues++; if (more) vtmp = vsep + 1; } while (more); if (val.vwildcards[0]) { ret = dmError(DMERR_INVALID_ARGS, "First grep value can not be a wildcard.\n"); goto out; } if (mode == FA_GREP) { // Check if we need to guess size if (val.type < 0) { for (int n = DMGV_last; n >= 0; n--) { const DMGrepType *def = &dmGrepTypes[n]; if (val.values[0] <= def->nmax) val.type = n; } } if (val.type < 0) { ret = dmError(DMERR_INVALID_ARGS, "Could not guess value type for '%s'.\n", arg); goto out; } // Check range for (int n = 0; n < val.nvalues; n++) if (!val.vwildcards[n] && val.values[n] > dmGrepTypes[val.type].nmax) { ret = dmError(DMERR_INVALID_ARGS, "Integer value %d <= %d <= %d out of range for type %s.\n", val.values[n], 0, dmGrepTypes[val.type].nmax, dmGrepTypes[val.type].name); goto out; } } else if (mode == FA_OFFSET) { if (val.type < 0) val.type = DMGV_UINT8; } if (nsetGrepValues < SET_MAX_VALUES) { DMGrepValue *node = &setGrepValues[nsetGrepValues++]; memcpy(node, &val, sizeof(val)); if (mode == FA_GREP) { dmPrint(1, "Grep %s: ", dmGrepTypes[val.type].name); dmPrintGrepValueList(node, FALSE, NULL, 0); } } else { ret = dmError(DMERR_BOUNDS, "Too many values specified (max %d).", SET_MAX_VALUES); } out: dmFree(vspec); dmFree(vstr); return ret; } BOOL argHandleOpt(const int optN, char *optArg, char *currArg) { (void) optArg; switch (optN) { case 0: argShowHelp(); exit(0); break; case 1: dmVerbosity++; break; case 2: return argParseGrepValue(optArg, FA_GREP) == DMERR_OK; case 3: return argParseGrepValue(optArg, FA_OFFSET) == DMERR_OK; default: dmErrorMsg("Unknown argument '%s'.\n", currArg); return FALSE; } return TRUE; } BOOL argHandleNonOpt(char *currArg) { if (nsrcFiles < SET_MAX_FILES) { DMSourceFile *file = &srcFiles[nsrcFiles++]; file->filename = currArg; return TRUE; } else { dmErrorMsg("Maximum number of input files exceeded (%d).\n", SET_MAX_FILES); return TRUE; } } void dmInitStats(DMStats *stats) { for (size_t n = 0; n < SET_MAX_ELEMS; n++) { stats->cv[n].count = 0; stats->cv[n].value = n; } } int dmCompareStatFunc(const void *va, const void *vb) { const DMStatValue *pa = va, *pb = vb; return pb->count - pa->count; } void dmPrintStats(DMStats *stats, const int nmax, const size_t size) { qsort(&stats->cv, SET_MAX_ELEMS, sizeof(DMStatValue), dmCompareStatFunc); for (int n = 0; n < nmax; n++) { printf("$%02x (%d = %1.2f%%), ", stats->cv[n].value, stats->cv[n].count, ((float) stats->cv[n].count * 100.0f) / (float) size); } printf("\n\n"); } int main(int argc, char *argv[]) { DMCompElem *compBuf = NULL; size_t compBufSize = 0, totalSize = 0; int res; dmInitProg("fanalyze", "Simple tool for file format analysis", "0.4", NULL, NULL); dmVerbosity = 1; dmInitStats(&totalStats); // Parse arguments if (!dmArgsProcess(argc, argv, optList, optListN, argHandleOpt, argHandleNonOpt, OPTH_BAILOUT)) exit(1); if (nsrcFiles < 1) { dmErrorMsg("Nothing to do. (try --help)\n"); goto out; } // Read input files for (int nfile = 0; nfile < nsrcFiles; nfile++) { DMSourceFile *file = &srcFiles[nfile]; if ((res = dmReadDataFile(NULL, file->filename, &file->data, &file->size)) != DMERR_OK) { dmErrorMsg("Could not read '%s': %s\n", file->filename, dmErrorStr(res)); goto out; } dmPrint(2, "Input #%d: '%s', %" DM_PRIu_SIZE_T " bytes.\n", nfile + 1, file->filename, file->size); if (!compBufSize || file->size < compBufSize) compBufSize = file->size; totalSize += file->size; dmInitStats(&file->stats); } // // Check what operating mode we are in // if (setMode == FA_GREP) { for (int nfile = 0; nfile < nsrcFiles; nfile++) { DMSourceFile *file = &srcFiles[nfile]; dmPrint(0, "\n%s\n", file->filename); for (int n = 0; n < nsetGrepValues; n++) { DMGrepValue *node = &setGrepValues[n]; const DMGrepType *def = &dmGrepTypes[node->type]; for (size_t offs = 0; offs + (def->bsize * node->nvalues) < file->size; offs++) { BOOL match = TRUE; for (int n = 0; n < node->nvalues; n++) if (!node->vwildcards[n]) { Uint32 mval; dmGetData(node->type, file, offs + n, &mval); if (mval != node->values[n]) { match = FALSE; break; } } if (match) { dmPrint(0, "%08x : ", offs); dmPrintGrepValueList(node, TRUE, file, offs); } } } } } else if (setMode == FA_OFFSET) { for (int nfile = 0; nfile < nsrcFiles; nfile++) { DMSourceFile *file = &srcFiles[nfile]; dmPrint(1, "#%03d: %s\n", nfile + 1, file->filename); } printf(" offset :"); for (int nfile = 0; nfile < nsrcFiles; nfile++) printf(" %03d ", nfile + 1); printf("\n"); printf("=========="); for (int nfile = 0; nfile < nsrcFiles; nfile++) printf("==========="); printf("\n"); for (int n = 0; n < nsetGrepValues; n++) { DMGrepValue *node = &setGrepValues[n]; const DMGrepType *def = &dmGrepTypes[node->type]; for (int nv = 0; nv < node->nvalues; nv++) { printf("%08x : ", node->values[nv]); for (int nfile = 0; nfile < nsrcFiles; nfile++) { DMSourceFile *file = &srcFiles[nfile]; Uint32 mval; char mstr[32]; int npad, nwidth; if (dmGetData(node->type, file, node->values[nv], &mval)) { char mfmt[16]; nwidth = def->bsize * 2; snprintf(mfmt, sizeof(mfmt), "%%0%d%s", nwidth, dmGrepDisp[node->disp].fmt); snprintf(mstr, sizeof(mstr), mfmt, mval); } else { strcpy(mstr, "----"); nwidth = 4; } npad = (10 - nwidth) / 2; for (int q = 0; q < npad; q++) fputc(' ', stdout); fputs(mstr, stdout); for (int q = 0; q < npad; q++) fputc(' ', stdout); } printf(" [%s]\n", dmGrepDisp[node->disp].name); } } } else if (setMode == FA_ANALYZE) { // Allocate comparision buffer // XXX: integer overflow? dmPrint(2, "Allocating %d element (%d bytes) comparision buffer.\n", compBufSize, compBufSize * sizeof(DMCompElem)); if ((compBuf = dmCalloc(compBufSize, sizeof(DMCompElem))) == NULL) { dmErrorMsg("Out of memory. Could not allocate comparision buffer!\n"); goto out; } // // Basic file data comparision // dmPrint(2, "Analyzing ..\n"); for (int nfile = 0; nfile < nsrcFiles; nfile++) { DMSourceFile *file = &srcFiles[nfile]; for (size_t offs = 0; offs < file->size; offs++) { Uint8 bv = file->data[offs]; totalStats.cv[bv].count++; file->stats.cv[bv].count++; } for (size_t offs = 0; offs < compBufSize; offs++) { Uint8 data = offs < file->size ? file->data[offs] : 0; compBuf[offs].stats[data]++; } } for (size_t offs = 0; offs < compBufSize; offs++) { DMCompElem *el = &compBuf[offs]; for (int n = 0; n < SET_MAX_ELEMS; n++) { if (el->stats[n] > 0) { el->variants++; el->data = n; } } } // // Display results // for (size_t offs = 0, n = 0; offs < compBufSize; offs++) { DMCompElem *el = &compBuf[offs]; BOOL var = el->variants > 1; if (n == 0) printf("%08" DM_PRIx_SIZE_T " | ", offs); if (var) printf("[%2d] ", el->variants); else printf(" %02x ", el->data); if (++n >= 16) { printf("\n"); n = 0; } } printf("\n"); // Attempt further analysis for (int nfile = 0; nfile < nsrcFiles; nfile++) { DMSourceFile *file = &srcFiles[nfile]; size_t len = file->size > compBufSize ? compBufSize : file->size; for (size_t offs = 0; offs + 4 < len; offs++) { DMCompElem *elem = &compBuf[offs]; for (int variant = 3; variant >= 0; variant--) { size_t nmax = (variant < 2) ? sizeof(Uint16) : sizeof(Uint32); Uint32 tmp = 0; for (size_t n = 0; n < nmax; n++) { size_t boffs = (variant & 1) ? n : nmax - n; tmp <<= 8; tmp |= file->data[offs + boffs]; } if (file->size - tmp < 32) { elem->interest[variant] += 32 - (file->size - tmp); elem->interestF[variant]++; } } } } printf("\nMore findings:\n"); for (size_t offs = 0; offs + 4 < compBufSize; offs++) { DMCompElem *elem = &compBuf[offs]; for (int variant = 0; variant < 4; variant++) if (elem->interestF[variant] > 0) { printf("%08" DM_PRIx_SIZE_T " | V%d : %d / %d\n", offs, variant, elem->interestF[variant], elem->interest[variant]); } } printf("\nGlobal most used bytes:\n"); dmPrintStats(&totalStats, 16, totalSize); for (int nfile = 0; nfile < nsrcFiles; nfile++) { DMSourceFile *file = &srcFiles[nfile]; printf("Most used bytes for '%s':\n", file->filename); dmPrintStats(&file->stats, 16, file->size); } } else { dmErrorMsg("Invalid operating mode?\n"); } out: for (int nfile = 0; nfile < nsrcFiles; nfile++) { DMSourceFile *file = &srcFiles[nfile]; dmFree(file->data); } return 0; }