Try using 4 means clustering.

src/nvtt/QuickCompressDXT.cpp

@@ -133,7 +133,7 @@ inline static float colorDistance(Vector3::Arg c0, Vector3::Arg c1)
 	return dot(c0-c1, c0-c1);
 }
 
-inline static uint computeIndices4(Vector3 block[16], Vector3::Arg maxColor, Vector3::Arg minColor)
+inline static uint computeIndices4(const Vector3 block[16], Vector3::Arg maxColor, Vector3::Arg minColor)
 {
 	Vector3 palette[4];
 	palette[0] = maxColor;
@@ -165,6 +165,28 @@ inline static uint computeIndices4(Vector3 block[16], Vector3::Arg maxColor, Vec
 	return indices;
 }
 
+inline static float evaluatePaletteError4(const Vector3 block[16], Vector3::Arg maxColor, Vector3::Arg minColor)
+{
+	Vector3 palette[4];
+	palette[0] = maxColor;
+	palette[1] = minColor;
+	palette[2] = lerp(palette[0], palette[1], 1.0f / 3.0f);
+	palette[3] = lerp(palette[0], palette[1], 2.0f / 3.0f);
+	
+	float total = 0.0f;
+	for (int i = 0; i < 16; i++)
+	{
+		float d0 = colorDistance(palette[0], block[i]);
+		float d1 = colorDistance(palette[1], block[i]);
+		float d2 = colorDistance(palette[2], block[i]);
+		float d3 = colorDistance(palette[3], block[i]);
+
+        total += min(min(d0, d1), min(d2, d3));
+	}
+
+	return total;
+}
+
 inline static uint computeIndices3(const ColorBlock & rgba, Vector3::Arg maxColor, Vector3::Arg minColor)
 {
 	Vector3 palette[4];
@@ -454,7 +476,7 @@ void QuickCompress::compressDXT1(const ColorBlock & rgba, BlockDXT1 * dxtBlock)
 		Vector3 block[16];
 		extractColorBlockRGB(rgba, block);
 
-#if 0		
+#if 1
 		// find min and max colors
 		Vector3 maxColor, minColor;
 		findMinMaxColorsBox(block, 16, &maxColor, &minColor);
@@ -462,15 +484,68 @@ void QuickCompress::compressDXT1(const ColorBlock & rgba, BlockDXT1 * dxtBlock)
 		selectDiagonal(block, 16, &maxColor, &minColor);
 		
 		insetBBox(&maxColor, &minColor);
-#endif
-		
+#else
 		float weights[16] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
 		Vector3 cluster[4];
-		Compute4Means(16, block, weights, Vector3(1, 1, 1), cluster);
-		
+		int count = Compute4Means(16, block, weights, Vector3(1, 1, 1), cluster);
+
 		Vector3 maxColor = cluster[0];
-		Vector3 minColor = cluster[3];
-		
+        Vector3 minColor = cluster[0];
+
+        float bestError = evaluatePaletteError4(block, maxColor, minColor);
+
+        if (count >= 2)
+        {
+            float error = evaluatePaletteError4(block, cluster[0], cluster[1]);
+            if (error < bestError) {
+                bestError = error;
+                maxColor = cluster[0];
+                minColor = cluster[1];
+            }
+
+            if (count >= 3)
+            {
+                error = evaluatePaletteError4(block, cluster[0], cluster[2]);
+                if (error < bestError) {
+                    bestError = error;
+                    maxColor = cluster[0];
+                    minColor = cluster[2];
+                }
+
+                error = evaluatePaletteError4(block, cluster[1], cluster[2]);
+                if (error < bestError) {
+                    bestError = error;
+                    maxColor = cluster[1];
+                    minColor = cluster[2];
+                }
+
+                if (count >= 4)
+                {
+                    error = evaluatePaletteError4(block, cluster[0], cluster[3]);
+                    if (error < bestError) {
+                        bestError = error;
+                        maxColor = cluster[0];
+                        minColor = cluster[3];
+                    }
+
+                    error = evaluatePaletteError4(block, cluster[1], cluster[3]);
+                    if (error < bestError) {
+                        bestError = error;
+                        maxColor = cluster[1];
+                        minColor = cluster[3];
+                    }
+
+                    error = evaluatePaletteError4(block, cluster[2], cluster[3]);
+                    if (error < bestError) {
+                        bestError = error;
+                        maxColor = cluster[2];
+                        minColor = cluster[3];
+                    }
+                }
+            }
+        }
+#endif
+
 		uint16 color0 = roundAndExpand(&maxColor);
 		uint16 color1 = roundAndExpand(&minColor);