Fix non-unit normals in mesh convex hull compiler.

PiperOrigin-RevId: 928777934
Change-Id: I43d7adcf4c5781b57f0af9805a8ce01ebc2e3872

doc/changelog.rst

@@ -44,6 +44,7 @@ General
 Bug fixes
 ^^^^^^^^^
 - Fixed a bug in the ``mjz`` :ref:`decoder <mjpDecoder>` where unnormalized paths would fail to be read.
+- Fixed a bug where the mesh compiler would produce non-unit convex hull polygon normals.
 
 Version 3.9.0 (May 27, 2026)
 ----------------------------

src/user/user_util.cc

@@ -379,6 +379,7 @@ template<typename T> double mjuu_makenormal(double* normal, const T a[3],
     normal[0] = 1;
     normal[1] = 0;
     normal[2] = 0;
+    return 1;
   }
   normal[0] /= nrm;
   normal[1] /= nrm;

src/user/user_util.h

@@ -115,8 +115,8 @@ void mjuu_localquat(double* local, const double* child, const double* parent);
 void mjuu_crossvec(double* a, const double* b, const double* c);
 
 // compute normal vector to given triangle
-template<typename T> double mjuu_makenormal(double* normal, const T a[3],
-                                            const T b[3], const T c[3]);
+template<typename T> MJAPI double mjuu_makenormal(double* normal, const T a[3],
+                                                  const T b[3], const T c[3]);
 
 // compute quaternion corresponding to minimal rotation from [0;0;1] to vec
 void mjuu_z2quat(double* quat, const double* vec);

test/user/user_util_test.cc

@@ -182,6 +182,26 @@ TEST_F(UserUtilTest, VectorToStringEmpty) {
   EXPECT_EQ(VectorToString(v), "");
 }
 
+TEST_F(UserUtilTest, MakeNormal) {
+  double a[3] = {0, 0, 0};
+  double b[3] = {1, 0, 0};
+  double c[3] = {0, 1, 0};
+  double normal[3];
+  double nrm = mjuu_makenormal(normal, a, b, c);
+  EXPECT_EQ(nrm, 1.0);
+  EXPECT_THAT(normal, ElementsAre(0, 0, 1));
+}
+
+TEST_F(UserUtilTest, MakeNormalDegenerate) {
+  double a[3] = {0, 0, 0};
+  double b[3] = {1, 0, 0};
+  double c[3] = {2, 0, 0};
+  double normal[3] = {0, 0, 0};
+  double nrm = mjuu_makenormal(normal, a, b, c);
+  EXPECT_EQ(nrm, 1.0);
+  EXPECT_THAT(normal, ElementsAre(1, 0, 0));
+}
+
 // utility: modified Gram-Schmidt to orthogonalize columns of Q (n x n)
 static void gramSchmidt(double* Q, int n) {
   for (int j = 0; j < n; j++) {