# HG changeset patch
# User Matti Hamalainen <ccr@tnsp.org>
# Date 1568618689 -10800
# Node ID 0085ce04788b2d157dd4ea8445ddedfc66927af0
# Parent  b57517a01960159636db1ab036252fa8bec08557
Add return value (error code) to dmGenericRLEAnalyze() and pass it through where used.

diff -r b57517a01960 -r 0085ce04788b tools/lib64fmts.c
--- a/tools/lib64fmts.c	Mon Sep 16 10:14:57 2019 +0300
+++ b/tools/lib64fmts.c	Mon Sep 16 10:24:49 2019 +0300
@@ -404,7 +404,8 @@
     cfg.rleMinCountB = 4;
     cfg.rleMaxCountB = 255;
 
-    dmGenericRLEAnalyze(&tmp, &cfg);
+    if ((res = dmGenericRLEAnalyze(&tmp, &cfg)) != DMERR_OK)
+        goto out;
 
     // Add the header bits
     if (!dmGrowBufPut(buf, (Uint8 *) magicID, strlen(magicID)) ||
@@ -530,7 +531,8 @@
     cfg.rleMinCountW = 256;
     cfg.rleMaxCountW = 1024;
 
-    dmGenericRLEAnalyze(&mem, &cfg);
+    if ((res = dmGenericRLEAnalyze(&mem, &cfg)) != DMERR_OK)
+        goto out;
 
     // Add the header bits
     if (!dmGrowBufPut(buf, (Uint8 *) fmtBDP5_MagicID, strlen(fmtBDP5_MagicID)) ||
@@ -747,7 +749,8 @@
     cfg.rleMinCountB = 4;
     cfg.rleMaxCountB = 256; // this format allows 256 byte runs with ZERO_COUNT_MAX
 
-    dmGenericRLEAnalyze(&tmp1, &cfg);
+    if ((res = dmGenericRLEAnalyze(&tmp1, &cfg)) != DMERR_OK)
+        goto out;
 
     if ((res = dmEncodeGenericRLEAlloc(&tmp2, &tmp1, &cfg)) != DMERR_OK)
         goto out;
@@ -1249,7 +1252,8 @@
     cfg.rleMinCountB = 4;
     cfg.rleMaxCountB = 255;
 
-    dmGenericRLEAnalyze(&mem, &cfg);
+    if ((res = dmGenericRLEAnalyze(&mem, &cfg)) != DMERR_OK)
+        goto out;
 
     // Add the header bits
     if (!dmGrowBufPut(buf, (Uint8 *) fmtFunPaint2_MagicID, strlen(fmtFunPaint2_MagicID)) ||
diff -r b57517a01960 -r 0085ce04788b tools/lib64gfx.c
--- a/tools/lib64gfx.c	Mon Sep 16 10:14:57 2019 +0300
+++ b/tools/lib64gfx.c	Mon Sep 16 10:24:49 2019 +0300
@@ -367,14 +367,14 @@
 }
 
 
-void dmGenericRLEAnalyze(const DMGrowBuf *buf, DMCompParams *cfg)
+int dmGenericRLEAnalyze(const DMGrowBuf *buf, DMCompParams *cfg)
 {
 #define DM_STAT_MAX 256
-    size_t *stats;
+    Uint8 *stats = NULL;
 
     // Allocate statistics counts buffer
-    if ((stats = dmMalloc0(DM_STAT_MAX * sizeof(size_t))) == NULL)
-        return;
+    if ((stats = dmMalloc0(DM_STAT_MAX * sizeof(Uint8))) == NULL)
+        return DMERR_MALLOC;
 
     // Get statistics on the data
     for (size_t offs = 0; offs < buf->len; offs++)
@@ -390,26 +390,24 @@
 
                 // Find least used byte value
                 for (size_t n = 0; n < DM_STAT_MAX; n++)
+                if (stats[n] < smallest)
                 {
-                    if (stats[n] < smallest)
+                    switch (cfg->flags & DM_RLE_RUNS_MASK)
                     {
-                        switch (cfg->flags & DM_RLE_RUNS_MASK)
-                        {
-                            case DM_RLE_BYTE_RUNS | DM_RLE_WORD_RUNS:
-                                cfg->rleMarkerW = selected;
-                                cfg->rleMarkerB = selected = n;
-                                break;
+                        case DM_RLE_BYTE_RUNS | DM_RLE_WORD_RUNS:
+                            cfg->rleMarkerW = selected;
+                            cfg->rleMarkerB = selected = n;
+                            break;
 
-                            case DM_RLE_BYTE_RUNS:
-                                cfg->rleMarkerB = selected = n;
-                                break;
+                        case DM_RLE_BYTE_RUNS:
+                            cfg->rleMarkerB = selected = n;
+                            break;
 
-                            case DM_RLE_WORD_RUNS:
-                                cfg->rleMarkerW = selected = n;
-                                break;
-                        }
-                        smallest = stats[n];
+                        case DM_RLE_WORD_RUNS:
+                            cfg->rleMarkerW = selected = n;
+                            break;
                     }
+                    smallest = stats[n];
                 }
             }
             break;
@@ -422,6 +420,7 @@
     }
 
     dmFree(stats);
+    return DMERR_OK;
 }
 
 
diff -r b57517a01960 -r 0085ce04788b tools/lib64gfx.h
--- a/tools/lib64gfx.h	Mon Sep 16 10:14:57 2019 +0300
+++ b/tools/lib64gfx.h	Mon Sep 16 10:24:49 2019 +0300
@@ -432,7 +432,7 @@
 int       dmC64ConvertGenericBMP2Image(DMImage *dst, const DMC64Image *src, const DMC64ImageConvSpec *spec);
 int       dmC64ConvertGenericImage2BMP(DMC64Image *dst, const DMImage *src, const DMC64ImageFormat *fmt, const DMC64ImageConvSpec *spec);
 
-void      dmGenericRLEAnalyze(const DMGrowBuf *buf, DMCompParams *cfg);
+int       dmGenericRLEAnalyze(const DMGrowBuf *buf, DMCompParams *cfg);
 
 void      dmSetupRLEBuffers(DMGrowBuf *dst, DMGrowBuf *src, const DMCompParams *cfg);
 void      dmFinishRLEBuffers(DMGrowBuf *dst, DMGrowBuf *src, const DMCompParams *cfg);