From 165ebc0db06b92149cb141fe4e32f443149b3803 Mon Sep 17 00:00:00 2001
From: Prasad <prasad@vtiger.com>
Date: Fri, 3 May 2024 23:37:48 +0530
Subject: [PATCH] Fixes #782: Ignore case and accents wheng grouping duplicate
records
---
include/utils/VtlibUtils.php | 32 +++++++++++++++++++++++++
modules/Vtiger/models/FindDuplicate.php | 8 +++----
2 files changed, 35 insertions(+), 5 deletions(-)
diff --git a/include/utils/VtlibUtils.php b/include/utils/VtlibUtils.php
index 87b4ba224..9f0390e74 100644
--- a/include/utils/VtlibUtils.php
+++ b/include/utils/VtlibUtils.php
@@ -744,6 +744,38 @@ function str_replace_json($search, $replace, $subject){
return json_decode(str_replace($search, $replace, json_encode($subject)), true);
}
+/**
+ * Case-insensitive comparision of string ignore accents.
+ * @param string $lv - left
+ * @param string $rv - right
+ * @return stcasecmp ascii comparision
+ */
+function strcasecmp_accents($lv, $rv) {
+ $lvenc = mb_detect_encoding($lv);
+ $rvenc = mb_detect_encoding($rv);
+ if ($lvenc != $rvenc) {
+ if ($lvenc != "ASCII") $lv = iconv($lvenc, "ASCII//TRANSLIT", $lv);
+ if ($rvenc != "ASCII") $rv = iconv($rvenc, "ASCII//TRANSLIT", $rv);
+ }
+ return strcasecmp($lv, $rv);
+}
+
+/**
+ * Callback function to use based on available environment support.
+ */
+function strcasecmp_accents_callback() {
+ // when mb & iconv is available - set the locale and return accents netural comparision
+ // otherwise return standard strcasecmp
+ if (function_exists('mb_detect_encoding') && function_exists('iconv')) {
+ setlocale(LC_CTYPE, "en_US.utf8"); // required to make iconv (UTF-8 to ASCII/TRANSLIT)
+ $callback = "strcasecmp_accents";
+ } else {
+ $callback = "strcasecmp";
+ }
+ return $callback;
+}
+
+
/**
* To purify malicious html event attributes
* @param <String> $value
diff --git a/modules/Vtiger/models/FindDuplicate.php b/modules/Vtiger/models/FindDuplicate.php
index a916c72db..892cb538d 100644
--- a/modules/Vtiger/models/FindDuplicate.php
+++ b/modules/Vtiger/models/FindDuplicate.php
@@ -69,7 +69,7 @@ class Vtiger_FindDuplicate_Model extends Vtiger_Base_Model {
$entries = array();
for($i=0; $i<$rows; $i++) {
// row will have value with (index and column names)
- $row = $db->query_result_rowdata($result, $i);
+ $row = $db->raw_query_result_rowdata($result, $i); // retrieve UTF-8 values.
// we should discard values with index for comparisions
$entries[] = array_filter($row, function($k) { return !is_numeric($k); }, ARRAY_FILTER_USE_KEY);
}
@@ -91,15 +91,13 @@ class Vtiger_FindDuplicate_Model extends Vtiger_Base_Model {
// make copy of current row
$slicedArray = array_slice($row, 0);
- // prepare for map comparisions
- array_walk($temp, 'lower_array');
- array_walk($slicedArray, 'lower_array');
unset($temp["recordid"]); // remove id which will obviously vary.
unset($slicedArray["recordid"]);
// if there is any value difference between (temp = prev) and (slicedArray = current)
// group them separately.
- $arrDiff = array_diff($temp, $slicedArray);
+ $arrDiff = array_udiff($temp, $slicedArray, strcasecmp_accents_callback()); // use case-less accent-less comparision.
+
if(php7_count($arrDiff) > 0) {
$groupCount++;
$temp = $slicedArray;
--
GitLab