Add single color compressor to optimal green compressor.

Improve quality of DXT1 green compressor increasing search range.

src/nvtt/OptimalCompressDXT.cpp

@@ -39,7 +39,22 @@ using namespace OptimalCompress;
 
 namespace
 {
-	static int computeGreenError(const ColorBlock & rgba, const BlockDXT1 * block)
+	static int greenDistance(int g0, int g1)
+	{
+		//return abs(g0 - g1);
+		int d = g0 - g1;
+		return d * d;
+	}
+
+	static int alphaDistance(int a0, int a1)
+	{
+		//return abs(a0 - a1);
+		int d = a0 - a1;
+		return d * d;
+	}
+
+
+	static int computeGreenError(const ColorBlock & rgba, const BlockDXT1 * block, int bestError = INT_MAX)
 	{
 		nvDebugCheck(block != NULL);
 
@@ -50,17 +65,22 @@ namespace
 		palette[3] = (2 * palette[1] + palette[0]) / 3;
 
 		int totalError = 0;
-
 		for (int i = 0; i < 16; i++)
 		{
 			const int green = rgba.color(i).g;
 			
-			int error = abs(green - palette[0]);
+			int error = greenDistance(green, palette[0]);
-			error = min(error, abs(green - palette[1]));
+			error = min(error, greenDistance(green, palette[1]));
-			error = min(error, abs(green - palette[2]));
+			error = min(error, greenDistance(green, palette[2]));
-			error = min(error, abs(green - palette[3]));
+			error = min(error, greenDistance(green, palette[3]));
-			
+
 			totalError += error;
+
+			if (totalError > bestError)
+			{
+				// early out
+				return totalError;
+			}
 		}
 
 		return totalError;
@@ -78,10 +98,10 @@ namespace
 		{
 			const int color = rgba.color(i).g;
 			
-			uint d0 = abs(color0 - color);
+			uint d0 = greenDistance(color0, color);
-			uint d1 = abs(color1 - color);
+			uint d1 = greenDistance(color1, color);
-			uint d2 = abs(color2 - color);
+			uint d2 = greenDistance(color2, color);
-			uint d3 = abs(color3 - color);
+			uint d3 = greenDistance(color3, color);
 			
 			uint b0 = d0 > d3;
 			uint b1 = d1 > d2;
@@ -102,49 +122,48 @@ namespace
 	// Choose quantized color that produces less error. Used by DXT3 compressor.
 	inline static uint quantize4(uint8 a)
 	{
-		int q0 = (a >> 4) - 1;
+		int q0 = max(int(a >> 4) - 1, 0);
 		int q1 = (a >> 4);
-		int q2 = (a >> 4) + 1;
+		int q2 = min(int(a >> 4) + 1, 0xF);
 		
 		q0 = (q0 << 4) | q0;
 		q1 = (q1 << 4) | q1;
 		q2 = (q2 << 4) | q2;
 		
-		int d0 = abs(q0 - a);
+		int d0 = alphaDistance(q0, a);
-		int d1 = abs(q1 - a);
+		int d1 = alphaDistance(q1, a);
-		int d2 = abs(q2 - a);
+		int d2 = alphaDistance(q2, a);
 
 		if (d0 < d1 && d0 < d2) return q0 >> 4;
 		if (d1 < d2) return q1 >> 4;
 		return q2 >> 4;
 	}
 	
-	static uint computeAlphaError(const ColorBlock & rgba, const AlphaBlockDXT5 * block)
+	static uint computeAlphaError(const ColorBlock & rgba, const AlphaBlockDXT5 * block, int bestError = INT_MAX)
 	{
 		uint8 alphas[8];
 		block->evaluatePalette(alphas);
 
-		uint totalError = 0;
+		int totalError = 0;
 
 		for (uint i = 0; i < 16; i++)
 		{
 			uint8 alpha = rgba.color(i).a;
 
-			uint besterror = 256*256;
+			int minDist = INT_MAX;
-			uint best;
 			for (uint p = 0; p < 8; p++)
 			{
-				int d = alphas[p] - alpha;
+				int dist = alphaDistance(alpha, alphas[p]);
-				uint error = d * d;
+				minDist = min(dist, minDist);
-
-				if (error < besterror)
-				{
-					besterror = error;
-					best = p;
-				}
 			}
 
-			totalError += besterror;
+			totalError += minDist;
+
+			if (totalError > bestError)
+			{
+				// early out
+				return totalError;
+			}
 		}
 
 		return totalError;
@@ -159,22 +178,21 @@ namespace
 		{
 			uint8 alpha = rgba.color(i).a;
 
-			uint besterror = 256*256;
+			int minDist = INT_MAX;
-			uint best = 8;
+			int bestIndex = 8;
-			for(uint p = 0; p < 8; p++)
+			for (uint p = 0; p < 8; p++)
 			{
-				int d = alphas[p] - alpha;
+				int dist = alphaDistance(alpha, alphas[p]);
-				uint error = d * d;
 
-				if (error < besterror)
+				if (dist < minDist)
 				{
-					besterror = error;
+					minDist = dist;
-					best = p;
+					bestIndex = p;
 				}
 			}
-			nvDebugCheck(best < 8);
+			nvDebugCheck(bestIndex < 8);
 
-			block->setIndex(i, best);
+			block->setIndex(i, bestIndex);
 		}
 	}
 
@@ -217,6 +235,23 @@ void OptimalCompress::compressDXT1a(Color32 rgba, BlockDXT1 * dxtBlock)
 	}
 }
 
+void OptimalCompress::compressDXT1G(uint8 g, BlockDXT1 * dxtBlock)
+{
+	dxtBlock->col0.r = 31;
+	dxtBlock->col0.g = OMatch6[g][0];
+	dxtBlock->col0.b = 0;
+	dxtBlock->col1.r = 31;
+	dxtBlock->col1.g = OMatch6[g][1];
+	dxtBlock->col1.b = 0;
+	dxtBlock->indices = 0xaaaaaaaa;
+
+	if (dxtBlock->col0.u < dxtBlock->col1.u)
+	{
+		swap(dxtBlock->col0.u, dxtBlock->col1.u);
+		dxtBlock->indices ^= 0x55555555;
+	}
+}
+
 
 // Brute force green channel compressor
 void OptimalCompress::compressDXT1G(const ColorBlock & rgba, BlockDXT1 * block)
@@ -226,12 +261,23 @@ void OptimalCompress::compressDXT1G(const ColorBlock & rgba, BlockDXT1 * block)
 	uint8 ming = 63;
 	uint8 maxg = 0;
 	
+	bool isSingleColor = true;
+	uint8 singleColor = rgba.color(0).g;
+
 	// Get min/max green.
 	for (uint i = 0; i < 16; i++)
 	{
 		uint8 green = rgba.color(i).g >> 2;
 		ming = min(ming, green);
 		maxg = max(maxg, green);
+
+		if (rgba.color(i).g != singleColor) isSingleColor = false;
+	}
+
+	if (isSingleColor)
+	{
+		compressDXT1G(singleColor, block);
+		return;
 	}
 
 	block->col0.r = 31;
@@ -241,36 +287,38 @@ void OptimalCompress::compressDXT1G(const ColorBlock & rgba, BlockDXT1 * block)
 	block->col0.b = 0;
 	block->col1.b = 0;
 
-	if (maxg - ming > 4)
+	int bestError = computeGreenError(rgba, block);
+	int bestg0 = maxg;
+	int bestg1 = ming;
+
+	// Expand search space a bit.
+	const int greenExpand = 4;
+	ming = (ming <= greenExpand) ? 0 : ming - greenExpand;
+	maxg = (maxg <= 63-greenExpand) ? 63 : maxg + greenExpand;
+
+	for (int g0 = ming+1; g0 <= maxg; g0++)
 	{
-		int besterror = computeGreenError(rgba, block);
+		for (int g1 = ming; g1 < g0; g1++)
-		int bestg0 = maxg;
-		int bestg1 = ming;
-		
-		for (int g0 = ming+5; g0 < maxg; g0++)
 		{
-			for (int g1 = ming; g1 < g0-4; g1++)
+			block->col0.g = g0;
+			block->col1.g = g1;
+			int error = computeGreenError(rgba, block, bestError);
+			
+			if (error < bestError)
 			{
-				if ((maxg-g0) + (g1-ming) > besterror)
+				bestError = error;
-					continue;
+				bestg0 = g0;
-				
+				bestg1 = g1;
-				block->col0.g = g0;
-				block->col1.g = g1;
-				int error = computeGreenError(rgba, block);
-				
-				if (error < besterror)
-				{
-					besterror = error;
-					bestg0 = g0;
-					bestg1 = g1;
-				}
 			}
 		}
-		
-		block->col0.g = bestg0;
-		block->col1.g = bestg1;
 	}
 	
+	block->col0.g = bestg0;
+	block->col1.g = bestg1;
+
+	nvDebugCheck(bestg0 == bestg1 || block->isFourColorMode());
+
+
 	Color32 palette[4];
 	block->evaluatePalette(palette);
 	block->indices = computeGreenIndices(rgba, palette);
@@ -313,42 +361,26 @@ void OptimalCompress::compressDXT5A(const ColorBlock & rgba, AlphaBlockDXT5 * dx
 	dxtBlock->alpha0 = maxa;
 	dxtBlock->alpha1 = mina;
 
-	/*int centroidDist = 256;
-	int centroid;
-
-	// Get the closest to the centroid.
-	for (uint i = 0; i < 16; i++)
-	{
-		uint8 alpha = rgba.color(i).a;
-		int dist = abs(alpha - (maxa + mina) / 2);
-		if (dist < centroidDist)
-		{
-			centroidDist = dist;
-			centroid = alpha;
-		}
-	}*/
-
 	if (maxa - mina > 8)
 	{
 		int besterror = computeAlphaError(rgba, dxtBlock);
 		int besta0 = maxa;
 		int besta1 = mina;
 
+		// Expand search space a bit.
+		const int alphaExpand = 8;
+		mina = (mina <= alphaExpand) ? 0 : mina - alphaExpand;
+		maxa = (maxa <= 255-alphaExpand) ? 255 : maxa + alphaExpand;
+
 		for (int a0 = mina+9; a0 < maxa; a0++)
 		{
 			for (int a1 = mina; a1 < a0-8; a1++)
-			//for (int a1 = mina; a1 < maxa; a1++)
 			{
-				//nvCheck(abs(a1-a0) > 8);
+				nvDebugCheck(a0 - a1 > 8);
-
-				//if (abs(a0 - a1) < 8) continue;
-				//if ((maxa-a0) + (a1-mina) + min(abs(centroid-a0), abs(centroid-a1)) > besterror)
-				if ((maxa-a0) + (a1-mina) > besterror)
-					continue;
 
 				dxtBlock->alpha0 = a0;
 				dxtBlock->alpha1 = a1;
-				int error = computeAlphaError(rgba, dxtBlock);
+				int error = computeAlphaError(rgba, dxtBlock, besterror);
 
 				if (error < besterror)
 				{

src/nvtt/OptimalCompressDXT.h

@@ -39,6 +39,7 @@ namespace nv
 	{
 		void compressDXT1(Color32 rgba, BlockDXT1 * dxtBlock);
 		void compressDXT1a(Color32 rgba, BlockDXT1 * dxtBlock);
+		void compressDXT1G(uint8 g, BlockDXT1 * dxtBlock);
 		
 		void compressDXT1G(const ColorBlock & rgba, BlockDXT1 * block);
 		void compressDXT3A(const ColorBlock & rgba, AlphaBlockDXT3 * dxtBlock);