Fixed error in CompressRGB, as reported by jonsoh in issue 49.

Added support for RGBA pixel format dithering. Fixes issue 50 and issue 51.
Add PixelFormat.h to vc8 project.

project/vc8/nvimage/nvimage.vcproj

@@ -355,6 +355,10 @@
 				RelativePath="..\..\..\src\nvimage\nvimage.h"
 				>
 			</File>
+			<File
+				RelativePath="..\..\..\src\nvimage\PixelFormat.h"
+				>
+			</File>
 			<File
 				RelativePath="..\..\..\src\nvimage\PsdFile.h"
 				>

src/nvimage/Quantize.cpp

@@ -16,6 +16,7 @@ http://www.efg2.com/Lab/Library/ImageProcessing/DHALF.TXT
 
 #include <nvimage/Image.h>
 #include <nvimage/Quantize.h>
+#include <nvimage/PixelFormat.h>
 
 using namespace nv;
 
@@ -47,94 +48,20 @@ void nv::Quantize::BinaryAlpha( Image * image, int alpha_threshold /*= 127*/ )
 // Simple quantization.
 void nv::Quantize::RGB16( Image * image )
 {
-	nvCheck(image != NULL);
+	Truncate(image, 5, 6, 5, 8);
-	
-	const uint w = image->width();
-	const uint h = image->height();
-	
-	for(uint y = 0; y < h; y++) {
-		for(uint x = 0; x < w; x++) {
-			
-			Color32 pixel32 = image->pixel(x, y);
-			
-			// Convert to 16 bit and back to 32 using regular bit expansion.
-			Color32 pixel16 = toColor32( toColor16(pixel32) );
-			
-			// Store color.
-			image->pixel(x, y) = pixel16;
-		}
-	}
 }
 
 // Alpha quantization.
 void nv::Quantize::Alpha4( Image * image )
 {
-	nvCheck(image != NULL);
+	Truncate(image, 8, 8, 8, 4);
-	
-	const uint w = image->width();
-	const uint h = image->height();
-	
-	for(uint y = 0; y < h; y++) {
-		for(uint x = 0; x < w; x++) {
-			
-			Color32 pixel = image->pixel(x, y);
-			
-			// Convert to 4 bit using regular bit expansion.
-			pixel.a = (pixel.a & 0xF0) | ((pixel.a & 0xF0) >> 4);
-			
-			// Store color.
-			image->pixel(x, y) = pixel;
-		}
-	}
 }
 
 
 // Error diffusion. Floyd Steinberg.
 void nv::Quantize::FloydSteinberg_RGB16( Image * image )
 {
-	nvCheck(image != NULL);
+	FloydSteinberg(image, 5, 6, 5, 8);
-	
-	const uint w = image->width();
-	const uint h = image->height();
-	
-	// @@ Use fixed point?
-	Vector3 * row0 = new Vector3[w+2];
-	Vector3 * row1 = new Vector3[w+2];
-	memset(row0, 0, sizeof(Vector3)*(w+2));
-	memset(row1, 0, sizeof(Vector3)*(w+2));
-	
-	for(uint y = 0; y < h; y++) {
-		for(uint x = 0; x < w; x++) {
-			
-			Color32 pixel32 = image->pixel(x, y);
-			
-			// Add error.	// @@ We shouldn't clamp here!
-			pixel32.r = clamp(int(pixel32.r) + int(row0[1+x].x()), 0, 255);
-			pixel32.g = clamp(int(pixel32.g) + int(row0[1+x].y()), 0, 255);
-			pixel32.b = clamp(int(pixel32.b) + int(row0[1+x].z()), 0, 255);
-			
-			// Convert to 16 bit. @@ Use regular clamp?
-			Color32 pixel16 = toColor32( toColor16(pixel32) );
-			
-			// Store color.
-			image->pixel(x, y) = pixel16;
-			
-			// Compute new error.
-			Vector3 diff(float(pixel32.r - pixel16.r), float(pixel32.g - pixel16.g), float(pixel32.b - pixel16.b));
-			
-			// Propagate new error.
-			row0[1+x+1] += 7.0f / 16.0f * diff;
-			row1[1+x-1] += 3.0f / 16.0f * diff;
-			row1[1+x+0] += 5.0f / 16.0f * diff;
-			row1[1+x+1] += 1.0f / 16.0f * diff;
-		}
-		
-		swap(row0, row1);
-		memset(row1, 0, sizeof(Vector3)*(w+2));
-	}
-	
-	delete [] row0;
-	delete [] row1;
 }
 
 
@@ -188,34 +115,90 @@ void nv::Quantize::FloydSteinberg_BinaryAlpha( Image * image, int alpha_threshol
 
 // Error diffusion. Floyd Steinberg.
 void nv::Quantize::FloydSteinberg_Alpha4( Image * image )
+{
+	FloydSteinberg(image, 8, 8, 8, 4);
+}
+
+
+void nv::Quantize::Truncate(Image * image, uint rsize, uint gsize, uint bsize, uint asize)
 {
 	nvCheck(image != NULL);
 	
 	const uint w = image->width();
 	const uint h = image->height();
 	
-	// @@ Use fixed point?
-	float * row0 = new float[(w+2)];
-	float * row1 = new float[(w+2)];
-	memset(row0, 0, sizeof(float)*(w+2));
-	memset(row1, 0, sizeof(float)*(w+2));
-	
 	for(uint y = 0; y < h; y++) {
 		for(uint x = 0; x < w; x++) {
 			
 			Color32 pixel = image->pixel(x, y);
 
-			// Add error.
+			// Convert to our desired size, and reconstruct.
-			int alpha = int(pixel.a) + int(row0[1+x]);
+			pixel.r = PixelFormat::convert(pixel.r, 8, rsize);
+			pixel.r = PixelFormat::convert(pixel.r, rsize, 8);
 
-			// Convert to 4 bit using regular bit expansion.
+			pixel.g = PixelFormat::convert(pixel.g, 8, gsize);
-			pixel.a = (pixel.a & 0xF0) | ((pixel.a & 0xF0) >> 4);
+			pixel.g = PixelFormat::convert(pixel.g, gsize, 8);
+
+			pixel.b = PixelFormat::convert(pixel.b, 8, bsize);
+			pixel.b = PixelFormat::convert(pixel.b, bsize, 8);
+
+			pixel.a = PixelFormat::convert(pixel.a, 8, asize);
+			pixel.a = PixelFormat::convert(pixel.a, asize, 8);
 
 			// Store color.
 			image->pixel(x, y) = pixel;
+		}
+	}
+}
+
+
+// Error diffusion. Floyd Steinberg.
+void nv::Quantize::FloydSteinberg(Image * image, uint rsize, uint gsize, uint bsize, uint asize)
+{
+	nvCheck(image != NULL);
+	
+	const uint w = image->width();
+	const uint h = image->height();
+	
+	Vector4 * row0 = new Vector4[w+2];
+	Vector4 * row1 = new Vector4[w+2];
+	memset(row0, 0, sizeof(Vector4)*(w+2));
+	memset(row1, 0, sizeof(Vector4)*(w+2));
+	
+	for (uint y = 0; y < h; y++) {
+		for (uint x = 0; x < w; x++) {
+			
+			Color32 pixel = image->pixel(x, y);
+
+			// Add error.
+			pixel.r = clamp(int(pixel.r) + int(row0[1+x].x()), 0, 255);
+			pixel.g = clamp(int(pixel.g) + int(row0[1+x].y()), 0, 255);
+			pixel.b = clamp(int(pixel.b) + int(row0[1+x].z()), 0, 255);
+			pixel.a = clamp(int(pixel.a) + int(row0[1+x].w()), 0, 255);
+			
+			int r = pixel.r;
+			int g = pixel.g;
+			int b = pixel.b;
+			int a = pixel.a;
+
+			// Convert to our desired size, and reconstruct.
+			r = PixelFormat::convert(r, 8, rsize);
+			r = PixelFormat::convert(r, rsize, 8);
+
+			g = PixelFormat::convert(g, 8, gsize);
+			g = PixelFormat::convert(g, gsize, 8);
+
+			b = PixelFormat::convert(b, 8, bsize);
+			b = PixelFormat::convert(b, bsize, 8);
+
+			a = PixelFormat::convert(a, 8, asize);
+			a = PixelFormat::convert(a, asize, 8);
+
+			// Store color.
+			image->pixel(x, y) = Color32(r, g, b, a);
 			
 			// Compute new error.
-			float diff = float(alpha - pixel.a);
+			Vector4 diff(float(int(pixel.r) - r), float(int(pixel.g) - g), float(int(pixel.b) - b), float(int(pixel.a) - a));
 			
 			// Propagate new error.
 			row0[1+x+1] += 7.0f / 16.0f * diff;
@@ -225,10 +208,9 @@ void nv::Quantize::FloydSteinberg_Alpha4( Image * image )
 		}
 		
 		swap(row0, row1);
-		memset(row1, 0, sizeof(float)*(w+2));
+		memset(row1, 0, sizeof(Vector4)*(w+2));
 	}
 	
 	delete [] row0;
 	delete [] row1;
 }
-

src/nvimage/Quantize.h

@@ -17,6 +17,9 @@ namespace nv
 		void FloydSteinberg_BinaryAlpha(Image * img, int alpha_threshold = 127);
 		void FloydSteinberg_Alpha4(Image * img);
 
+		void Truncate(Image * image, uint rsize, uint gsize, uint bsize, uint asize);
+		void FloydSteinberg(Image * image, uint rsize, uint gsize, uint bsize, uint asize);
+
 		// @@ Add palette quantization algorithms!
 	}
 }

src/nvtt/CompressRGB.cpp

@@ -123,7 +123,7 @@ void nv::compressRGB(const Image * image, const OutputOptions::Private & outputO
 			}
 			
 			// Zero padding.
-			for (uint x = w; x < pitch; x++)
+			for (uint x = w * byteCount; x < pitch; x++)
 			{
 				*(dst + x) = 0;
 			}

src/nvtt/Compressor.cpp

@@ -34,6 +34,7 @@
 #include <nvimage/Filter.h>
 #include <nvimage/Quantize.h>
 #include <nvimage/NormalMap.h>
+#include <nvimage/PixelFormat.h>
 
 #include "Compressor.h"
 #include "InputOptions.h"
@@ -617,13 +618,6 @@ void Compressor::Private::quantizeMipmap(Mipmap & mipmap, const CompressionOptio
 {
 	nvDebugCheck(mipmap.asFixedImage() != NULL);
 
-	if (compressionOptions.enableColorDithering)
-	{
-		if (compressionOptions.format >= Format_DXT1 && compressionOptions.format <= Format_DXT5)
-		{
-			Quantize::FloydSteinberg_RGB16(mipmap.asMutableFixedImage());
-		}
-	}
 	if (compressionOptions.binaryAlpha)
 	{
 		if (compressionOptions.enableAlphaDithering)
@@ -635,19 +629,50 @@ void Compressor::Private::quantizeMipmap(Mipmap & mipmap, const CompressionOptio
 			Quantize::BinaryAlpha(mipmap.asMutableFixedImage(), compressionOptions.alphaThreshold);
 		}
 	}
-	else
+
+	if (compressionOptions.enableColorDithering || compressionOptions.enableAlphaDithering)
 	{
+		uint rsize = 8;
+		uint gsize = 8;
+		uint bsize = 8;
+		uint asize = 8;
+
+		if (compressionOptions.enableColorDithering)
+		{
+			if (compressionOptions.format >= Format_DXT1 && compressionOptions.format <= Format_DXT5)
+			{
+				rsize = 5;
+				gsize = 6;
+				bsize = 5;
+			}
+			else if (compressionOptions.format == Format_RGB)
+			{
+				uint rshift, gshift, bshift;
+				PixelFormat::maskShiftAndSize(compressionOptions.rmask, &rshift, &rsize);
+				PixelFormat::maskShiftAndSize(compressionOptions.gmask, &gshift, &gsize);
+				PixelFormat::maskShiftAndSize(compressionOptions.bmask, &bshift, &bsize);
+			}
+		}
+
 		if (compressionOptions.enableAlphaDithering)
 		{
 			if (compressionOptions.format == Format_DXT3)
 			{
-				Quantize::Alpha4(mipmap.asMutableFixedImage());
+				asize = 4;
 			}
-			else if (compressionOptions.format == Format_DXT1a)
+			else if (compressionOptions.format == Format_RGB)
 			{
-				Quantize::BinaryAlpha(mipmap.asMutableFixedImage(), compressionOptions.alphaThreshold);
+				uint ashift;
+				PixelFormat::maskShiftAndSize(compressionOptions.amask, &ashift, &asize);
 			}
 		}
+
+		if (compressionOptions.binaryAlpha)
+		{
+			asize = 8; // Already quantized.
+		}
+
+		Quantize::FloydSteinberg(mipmap.asMutableFixedImage(), rsize, gsize, bsize, asize);
 	}
 }
 

src/nvtt/tools/compress.cpp

@@ -373,7 +373,6 @@ int main(int argc, char *argv[])
 		inputOptions.setMipmapGeneration(false);
 	}
 
-
 	nvtt::CompressionOptions compressionOptions;
 	compressionOptions.setFormat(format);
 	if (fast)
@@ -397,6 +396,21 @@ int main(int argc, char *argv[])
 		compressionOptions.setExternalCompressor(externalCompressor);
 	}
 
+	if (format == nvtt::Format_RGB)
+	{
+		compressionOptions.setQuantization(true, false, false);
+		//compressionOptions.setPixelFormat(16, 0xF000, 0x0F00, 0x00F0, 0x000F);
+		compressionOptions.setPixelFormat(16, 
+			0x0F00, 
+			0x00F0, 
+			0x000F, 
+			0xF000);
+		//	0x003F0000, 
+		//	0x00003F00, 
+		//	0x0000003F, 
+		//	0x3F000000);
+	}
+
 	
 	MyErrorHandler errorHandler;
 	MyOutputHandler outputHandler(output);