CVE-2009-4134,CVE-2010-1449,CVE-2010-1450-rgbimg-int-overflow.patch   [plain text]


--- Modules/rgbimgmodule.c.0	2008-08-19 13:57:46.000000000 -0700
+++ Modules/rgbimgmodule.c	2010-06-17 17:05:57.000000000 -0700
@@ -20,13 +20,8 @@
 typedef int Py_Int32;
 typedef unsigned int Py_UInt32;
 #else
-#if SIZEOF_LONG == 4
-typedef long Py_Int32;
-typedef unsigned long Py_UInt32;
-#else
 #error "No 4-byte integral type"
 #endif
-#endif
 
 #include <string.h>
 
@@ -92,7 +87,7 @@ typedef struct {
 
 #define CHANOFFSET(z)	(3-(z))	/* this is byte order dependent */
 
-static void expandrow(unsigned char *, unsigned char *, int);
+static int expandrow(unsigned char *, unsigned char *, int, int);
 static void setalpha(unsigned char *, int);
 static void copybw(Py_Int32 *, int);
 static void interleaverow(unsigned char*, unsigned char*, int, int);
@@ -269,7 +264,7 @@ longimagedata(PyObject *self, PyObject *
 	Py_Int32 *starttab = NULL, *lengthtab = NULL;
 	FILE *inf = NULL;
 	IMAGE image;
-	int y, z, tablen, new_size;
+	int y, z, tablen;
 	int xsize, ysize, zsize;
 	int bpp, rle, cur, badorder;
 	int rlebuflen;
@@ -299,22 +294,15 @@ longimagedata(PyObject *self, PyObject *
 	xsize = image.xsize;
 	ysize = image.ysize;
 	zsize = image.zsize;
-	tablen = xsize * ysize * zsize * sizeof(Py_Int32);
-        if (xsize != (((tablen / ysize) / zsize) / sizeof(Py_Int32))) {
-		PyErr_NoMemory();
-		goto finally;
-        }
+	if (image.zsize > 4)
+		goto corrupted;
+
 	if (rle) {
+		/* these can't oveflow, as xsize and ysize <= 0xffff and zsize <= 4 */
 		tablen = ysize * zsize * sizeof(Py_Int32);
-		rlebuflen = (int) (1.05 * xsize +10);
-		if ((tablen / sizeof(Py_Int32)) != (ysize * zsize) ||
-		    rlebuflen < 0) {
-			PyErr_NoMemory();
-			goto finally;
-		}
-
 		starttab = (Py_Int32 *)malloc(tablen);
 		lengthtab = (Py_Int32 *)malloc(tablen);
+		rlebuflen = (int) (1.05 * xsize +10);
 		rledat = (unsigned char *)malloc(rlebuflen);
 		if (!starttab || !lengthtab || !rledat) {
 			PyErr_NoMemory();
@@ -342,14 +330,12 @@ longimagedata(PyObject *self, PyObject *
 
 		fseek(inf, 512 + 2 * tablen, SEEK_SET);
 		cur = 512 + 2 * tablen;
-		new_size = xsize * ysize + TAGLEN;
-		if (new_size < 0 || (new_size * sizeof(Py_Int32)) < 0) {
+		if (ysize > 0 && xsize > (INT_MAX/sizeof(Py_Int32)-TAGLEN)/ysize) {
 			PyErr_NoMemory();
 			goto finally;
 		}
-
 		rv = PyString_FromStringAndSize((char *)NULL,
-				      new_size * sizeof(Py_Int32));
+				      (xsize * ysize + TAGLEN) * sizeof(Py_Int32));
 		if (rv == NULL)
 			goto finally;
 
@@ -371,15 +357,13 @@ longimagedata(PyObject *self, PyObject *
 						cur = starttab[idx];
 					}
 					if (lengthtab[idx] > rlebuflen) {
-						PyErr_SetString(ImgfileError,
-							"rlebuf is too small");
-						Py_DECREF(rv);
-						rv = NULL;
-						goto finally;
+						PyErr_SetString(ImgfileError, "rlebuf is too small");
+						goto corrupted;
 					}
 					fread(rledat, lengthtab[idx], 1, inf);
 					cur += lengthtab[idx];
-					expandrow(lptr, rledat, 3-z);
+					if (expandrow(lptr, rledat, 3-z, xsize) != 0)
+						goto corrupted;
 					if (reverse_order)
 						lptr -= xsize
 						      * sizeof(Py_UInt32);
@@ -401,9 +385,14 @@ longimagedata(PyObject *self, PyObject *
 						      SEEK_SET);
 						cur = starttab[idx];
 					}
+					if (lengthtab[idx] > rlebuflen) {
+						PyErr_SetString(ImgfileError, "rlebuf is too small");
+						goto corrupted;
+					}
 					fread(rledat, lengthtab[idx], 1, inf);
 					cur += lengthtab[idx];
-					expandrow(lptr, rledat, 3-z);
+					if (expandrow(lptr, rledat, 3-z, xsize) != 0)
+						goto corrupted;
 				}
 				if (reverse_order)
 					lptr -= xsize * sizeof(Py_UInt32);
@@ -417,14 +406,12 @@ longimagedata(PyObject *self, PyObject *
 			copybw((Py_Int32 *) base, xsize * ysize);
 	}
 	else {
-		new_size = xsize * ysize + TAGLEN;
-		if (new_size < 0 || (new_size * sizeof(Py_Int32)) < 0) {
+		if (ysize > 0 && xsize > (INT_MAX/sizeof(Py_Int32)-TAGLEN)/ysize) {
 			PyErr_NoMemory();
 			goto finally;
 		}
-
 		rv = PyString_FromStringAndSize((char *) 0,
-						new_size*sizeof(Py_Int32));
+					   (xsize*ysize+TAGLEN)*sizeof(Py_Int32));
 		if (rv == NULL)
 			goto finally;
 
@@ -458,6 +445,14 @@ longimagedata(PyObject *self, PyObject *
 		else if (zsize < 3) 
 			copybw((Py_Int32 *) base, xsize * ysize);
 	}
+	goto finally;
+  corrupted:
+	PyErr_SetString(ImgfileError,
+			"input image file is corrupted");
+	if (rv) {
+		Py_DECREF(rv);
+		rv = NULL;
+	}
   finally:
 	if (starttab)
 		free(starttab);
@@ -525,8 +520,8 @@ setalpha(unsigned char *lptr, int n)
 	}
 }
 
-static void
-expandrow(unsigned char *optr, unsigned char *iptr, int z)
+static int
+expandrow(unsigned char *optr, unsigned char *iptr, int z, int n)
 {
 	unsigned char pixel, count;
 
@@ -534,7 +529,12 @@ expandrow(unsigned char *optr, unsigned 
 	while (1) {
 		pixel = *iptr++;
 		if (!(count = (pixel & 0x7f)))
-			return;
+			return 0;
+		/* don't write more than n byte to output */
+		if (count > n)
+			return 1;
+		else
+			n -= count;
 		if (pixel & 0x80) {
 			while (count >= 8) {
 				optr[0 * 4] = iptr[0];
@@ -606,6 +606,14 @@ longstoimage(PyObject *self, PyObject *a
 	if (!PyArg_ParseTuple(args, "s#iiis:longstoimage", &lptr, &len,
 			      &xsize, &ysize, &zsize, &name))
 		return NULL;
+	
+	if (xsize < 0 || xsize > 0xffff
+			||  ysize < 0 || ysize > 0xffff
+			||  zsize < 0 || zsize > 4) {
+		PyErr_SetString(ImgfileError, "invalid xsize, ysize, or zsize");
+		return NULL;
+	}
+
 
 	goodwrite = 1;
 	outf = fopen(name, "wb");
@@ -613,17 +621,12 @@ longstoimage(PyObject *self, PyObject *a
 		PyErr_SetString(ImgfileError, "can't open output file");
 		return NULL;
 	}
-	tablen = ysize * zsize * sizeof(Py_Int32);
-	rlebuflen = (int) (1.05 * xsize + 10);
-
-	if ((tablen / sizeof(Py_Int32)) != (ysize * zsize) ||
-	    rlebuflen < 0 || (xsize * sizeof(Py_Int32)) < 0) {
-		PyErr_NoMemory();
-		goto finally;
-	}
 
+	/* these can't oveflow, as xsize and ysize <= 0xffff and zsize <= 4 */
+	tablen = ysize * zsize * sizeof(Py_Int32);
 	starttab = (Py_Int32 *)malloc(tablen);
 	lengthtab = (Py_Int32 *)malloc(tablen);
+	rlebuflen = (int) (1.05 * xsize + 10);
 	rlebuf = (unsigned char *)malloc(rlebuflen);
 	lumbuf = (unsigned char *)malloc(xsize * sizeof(Py_Int32));
 	if (!starttab || !lengthtab || !rlebuf || !lumbuf) {