Cleanup and bugfix FindEndpoints

src/BC1/BC1Encoder.cpp

@@ -184,8 +184,8 @@ void BC1Encoder::FindEndpoints(Color4x4 pixels, BC1Encoder::Flags flags, const B
 
         if (metrics.max.r - metrics.min.r < 2) {
             // single color block
-            low.r = high.r = scale8To5(fr);
-            low.g = high.g = scale8To6(fr);
+            low.r = high.r = (uint8_t)scale8To5(fr);
+            low.g = high.g = (uint8_t)scale8To6(fr);
             low.b = high.b = low.r;
         } else {
             low.r = low.b = scale8To5(metrics.min.r);
@@ -204,7 +204,7 @@ void BC1Encoder::FindEndpoints(Color4x4 pixels, BC1Encoder::Flags flags, const B
         auto &min = metrics.min;
         auto &max = metrics.max;
 
-        unsigned chan0 = diff.MaxChannelRGB();  // primary axis of the bounding box
+        unsigned chan0 = (unsigned)diff.MaxChannelRGB();  // primary axis of the bounding box
         l[chan0] = (float)min[chan0];
         h[chan0] = (float)min[chan0];
 
@@ -223,7 +223,7 @@ void BC1Encoder::FindEndpoints(Color4x4 pixels, BC1Encoder::Flags flags, const B
         float denominator = (float)(16 * sum_xx) - (float)(sum_x * sum_x);
 
         // once per secondary axis, calculate high and low using least squares
-        if (fabs(denominator > 1e-8f)) {
+        if (fabs(denominator) > 1e-8f) {
             for (unsigned i = 1; i < 3; i++) {
                 /* each secondary axis is fitted with a linear formula of the form
                  *  y = ax + b
@@ -286,8 +286,6 @@ void BC1Encoder::FindEndpoints(Color4x4 pixels, BC1Encoder::Flags flags, const B
 
         Color min, max;
 
-        const float bias = 8.0f / 255.0f;
-
         // rescale and inset values
         for (unsigned c = 0; c < 3; c++) {
             int inset = ((int)(metrics.max[c] - metrics.min[c]) - 8) >> 4;  // 1/16 of delta, with bias
@@ -317,7 +315,7 @@ void BC1Encoder::FindEndpoints(Color4x4 pixels, BC1Encoder::Flags flags, const B
 
         std::array<Vector4, 16> colors;
 
-        Vector4 axis = {306, 601, 117};  // I think this is luma?
+        Vector4 axis = {306, 601, 117};  // Luma vector
         Matrix4x4 covariance;
         const unsigned total_power_iters = (flags & Flags::Use6PowerIters) != Flags::None ? 6 : 4;
 
@@ -341,8 +339,10 @@ void BC1Encoder::FindEndpoints(Color4x4 pixels, BC1Encoder::Flags flags, const B
         if (covariance[0][2] < 0) delta[0] = -delta[0];  // r vs b
         if (covariance[1][2] < 0) delta[1] = -delta[1];  // g vs b
 
+        // using the covariance matrix, iteratively stretch the delta vector towards the primary axis of the data
         for (unsigned power_iter = 0; power_iter < total_power_iters; power_iter++) { delta = covariance * delta; }
 
+        // if we found any correlation, then this is our new axis. otherwise we fallback to the luma vector
         float k = delta.MaxAbs(3);
         if (k > 2) { axis = delta * (2048.0f / k); }
 
@@ -352,6 +352,8 @@ void BC1Encoder::FindEndpoints(Color4x4 pixels, BC1Encoder::Flags flags, const B
         unsigned min_index, max_index;
 
         for (unsigned i = 0; i < 16; i++) {
+            // since axis is constant here, I dont think its magnitude actually matters,
+            // since we only care about the min or max dot product
             float dot = colors[i].Dot(axis);
             if (dot > max_dot) {
                 max_dot = dot;