Fixes #782: Ignore case and accents wheng grouping duplicate records

include/utils/VtlibUtils.php

@@ -744,6 +744,38 @@ function str_replace_json($search, $replace, $subject){
     return json_decode(str_replace($search, $replace,  json_encode($subject)), true);
 }
 
+/**
+ * Case-insensitive comparision of string ignore accents.
+ * @param string $lv - left
+ * @param string $rv - right
+ * @return stcasecmp ascii comparision
+ */
+function strcasecmp_accents($lv, $rv) {
+    $lvenc = mb_detect_encoding($lv);
+    $rvenc = mb_detect_encoding($rv);
+    if ($lvenc != $rvenc) {
+        if ($lvenc != "ASCII") $lv = iconv($lvenc, "ASCII//TRANSLIT", $lv);
+        if ($rvenc != "ASCII") $rv = iconv($rvenc, "ASCII//TRANSLIT", $rv);
+    }
+    return strcasecmp($lv, $rv);
+}
+
+/**
+ * Callback function to use based on available environment support.
+ */
+function strcasecmp_accents_callback() {
+    // when mb & iconv is available - set the locale and return accents netural comparision
+    // otherwise return standard strcasecmp
+    if (function_exists('mb_detect_encoding') && function_exists('iconv')) {
+        setlocale(LC_CTYPE, "en_US.utf8"); // required to make iconv (UTF-8 to ASCII/TRANSLIT)
+        $callback = "strcasecmp_accents";
+    } else {
+        $callback = "strcasecmp";
+    }
+    return $callback;
+}
+
+
 /**
  * To purify malicious html event attributes
  * @param <String> $value

modules/Vtiger/models/FindDuplicate.php

@@ -69,7 +69,7 @@ class Vtiger_FindDuplicate_Model extends Vtiger_Base_Model {
         $entries = array();
         for($i=0; $i<$rows; $i++) {
             // row will have value with (index and column names)
-            $row = $db->query_result_rowdata($result, $i);
+            $row = $db->raw_query_result_rowdata($result, $i); // retrieve UTF-8 values.
             // we should discard values with index for comparisions
 			$entries[] = array_filter($row, function($k) { return !is_numeric($k); }, ARRAY_FILTER_USE_KEY);
 		}
@@ -91,15 +91,13 @@ class Vtiger_FindDuplicate_Model extends Vtiger_Base_Model {
                 // make copy of current row
                 $slicedArray = array_slice($row, 0);
 
-                // prepare for map comparisions
-                array_walk($temp, 'lower_array');
-                array_walk($slicedArray, 'lower_array');
                 unset($temp["recordid"]); // remove id which will obviously vary.
                 unset($slicedArray["recordid"]);
 
                 // if there is any value difference between (temp = prev) and (slicedArray = current) 
                 // group them separately.
-                $arrDiff = array_diff($temp, $slicedArray);
+				$arrDiff = array_udiff($temp, $slicedArray, strcasecmp_accents_callback()); // use case-less accent-less comparision.
+				
                 if(php7_count($arrDiff) > 0) {
                     $groupCount++;
                     $temp = $slicedArray;