bug#81762: [PATCH] join,uniq: fold multibyte letters with -i
From:
Iván Ezequiel Rodriguez
Subject:
bug#81762: [PATCH] join,uniq: fold multibyte letters with -i
Date:
Mon, 31 Aug 2026 14:30:22 -0300
join -i and uniq -i used memcasecmp, which only folds ASCII A-Z, so
in UTF-8 locales É and é did not match despite -i. That matches the
long-standing FIXME in join's keycmp.
In multibyte locales, compare c32tolower of each mcel character
(same approach as gnulib exclude). Unibyte locales keep the historical
memcasecmp + length behavior. As before, -i does not use LC_COLLATE.
This is simple codepoint folding, not full Unicode case mapping:
É matches é, but ß and SS remain distinct.
* src/memcasecmp-mcel.h (memcasecmp_fields): New shared helper.
* src/join.c (keycmp): Use it for -i.
* src/uniq.c (different): Likewise.
* src/local.mk: List the new header.
* tests/join/join.pl: Cover É/é join and ß/SS non-join.
* tests/uniq/uniq.pl: Cover É/é collapse and ß/SS distinct.
* NEWS: Mention the improvement.
Signed-off-by: Iván Ezequiel Rodriguez <ivanrwcm25@gmail.com>
---
NEWS | 5 ++++
src/join.c | 17 +++++---------
src/local.mk | 1 +
src/memcasecmp-mcel.h | 54 +++++++++++++++++++++++++++++++++++++++++++
src/uniq.c | 6 ++---
tests/join/join.pl | 18 +++++++++++++++
tests/uniq/uniq.pl | 13 ++++++++++-
7 files changed, 99 insertions(+), 15 deletions(-)
create mode 100644 src/memcasecmp-mcel.h
diff --git a/NEWS b/NEWS
index f1040bde5..15873e90a 100644
--- a/NEWS
+++ b/NEWS
@@ -97,6 +97,11 @@ GNU coreutils NEWS -*-
outline -*-
stat -f -c%T now reports the file system type,
and tail -f uses inotify for this file system.
+ 'join -i' and 'uniq -i' fold multi-byte letters with c32tolower in
+ multibyte locales (e.g. É matches é), instead of only ASCII via
+ memcasecmp. This is simple codepoint folding, not full Unicode case
+ mapping (ß and SS remain distinct).
+
uname adds the -A,--all-labeled option to label all output, one item per
line.
** Changes in behavior
diff --git a/src/join.c b/src/join.c
index 4346758a6..0cb8e1640 100644
--- a/src/join.c
+++ b/src/join.c
@@ -28,6 +28,7 @@
#include "linebuffer.h"
#include "mcel.h"
#include "memcasecmp.h"
+#include "memcasecmp-mcel.h"
#include "quote.h"
#include "skipchars.h"
#include "stdio--.h"
@@ -397,17 +398,11 @@ keycmp (struct line const *line1, struct line const
*line2,
return 1;
if (ignore_case)
- {
- /* FIXME: ignore_case does not work with NLS (in particular,
- with multibyte chars). */
- diff = memcasecmp (beg1, beg2, MIN (len1, len2));
- }
- else
- {
- if (hard_LC_COLLATE)
- return xmemcoll (beg1, len1, beg2, len2);
- diff = memcmp (beg1, beg2, MIN (len1, len2));
- }
+ return memcasecmp_fields (beg1, len1, beg2, len2);
+
+ if (hard_LC_COLLATE)
+ return xmemcoll (beg1, len1, beg2, len2);
+ diff = memcmp (beg1, beg2, MIN (len1, len2));
if (diff)
return diff;
diff --git a/src/local.mk b/src/local.mk
index 6ed4ed78f..2694b92a9 100644
--- a/src/local.mk
+++ b/src/local.mk
@@ -56,6 +56,7 @@ noinst_HEADERS = \
src/iopoll.h \
src/longlong.h \
src/ls.h \
+ src/memcasecmp-mcel.h \
src/octhexdigits.h \
src/operand2sig.h \
src/printenv.h \
diff --git a/src/memcasecmp-mcel.h b/src/memcasecmp-mcel.h
new file mode 100644
index 000000000..c69e24811
--- /dev/null
+++ b/src/memcasecmp-mcel.h
@@ -0,0 +1,54 @@
+/* Case-insensitive compare of memory regions using mcel / c32tolower.
+
+ Copyright (C) 2026 Free Software Foundation, Inc.
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>. */
+
+/* Include after system.h, mcel.h, and memcasecmp.h. */
+
+/* Compare S1[0..N1) and S2[0..N2) ignoring case.
+ Unibyte locales: historical memcasecmp + length.
+ Multibyte: compare c32tolower of each mcel character so equal-folded
+ fields match even when UTF-8 byte lengths differ. This is simple
+ codepoint case folding, not full Unicode case mapping (e.g. ß vs SS
+ remain distinct). Does not consult LC_COLLATE. */
+ATTRIBUTE_PURE
+static inline int
+memcasecmp_fields (char const *s1, idx_t n1, char const *s2, idx_t n2)
+{
+ if (MB_CUR_MAX <= 1)
+ {
+ int diff = memcasecmp (s1, s2, MIN (n1, n2));
+ return diff ? diff : _GL_CMP (n1, n2);
+ }
+
+ char const *p1 = s1;
+ char const *e1 = s1 + n1;
+ char const *p2 = s2;
+ char const *e2 = s2 + n2;
+
+ while (p1 < e1 && p2 < e2)
+ {
+ mcel_t g1 = mcel_scan (p1, e1);
+ mcel_t g2 = mcel_scan (p2, e2);
+ char32_t c1 = g1.err ? to_uchar (*p1) : c32tolower (g1.ch);
+ char32_t c2 = g2.err ? to_uchar (*p2) : c32tolower (g2.ch);
+ if (c1 != c2)
+ return _GL_CMP (c1, c2);
+ p1 += g1.err ? 1 : g1.len;
+ p2 += g2.err ? 1 : g2.len;
+ }
+
+ return (p1 < e1) - (p2 < e2);
+}
diff --git a/src/uniq.c b/src/uniq.c
index f91c4212c..9f4d5697b 100644
--- a/src/uniq.c
+++ b/src/uniq.c
@@ -32,6 +32,7 @@
#include "stdio--.h"
#include "xstrtol.h"
#include "memcasecmp.h"
+#include "memcasecmp-mcel.h"
#include "quote.h"
/* The official name of this program (e.g., no 'g' prefix). */
@@ -304,9 +305,8 @@ static bool
different (char *old, char *new, idx_t oldlen, idx_t newlen)
{
if (ignore_case)
- return oldlen != newlen || memcasecmp (old, new, oldlen);
- else
- return oldlen != newlen || memcmp (old, new, oldlen);
+ return memcasecmp_fields (old, oldlen, new, newlen) != 0;
+ return oldlen != newlen || memcmp (old, new, oldlen);
}
/* Output the line in linebuffer LINE to standard output
diff --git a/tests/join/join.pl b/tests/join/join.pl
index 1bc6d1305..6345f88ce 100755
--- a/tests/join/join.pl
+++ b/tests/join/join.pl
@@ -401,6 +401,24 @@ if ($mb_locale ne 'C')
push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}];
}
push @Tests, @new;
+
+ # -i with UTF-8 letters: memcasecmp is byte-wise and misses these.
+ # Added after the -mb duplication to avoid a second ENV spec.
+ # É (U+00C9) / é (U+00E9) fold together; ß (U+00DF) / SS do not
+ # (c32tolower is not full Unicode case mapping).
+ push @Tests,
+ ['i-utf8-join', '-i',
+ {IN=>"\xC3\x89 a\n"}, {IN=>"\xC3\xA9 b\n"},
+ {OUT=>"\xC3\x89 a b\n"},
+ {ENV => "LC_ALL=$mb_locale"}],
+ ['i-utf8-nojoin', '',
+ {IN=>"\xC3\x89 a\n"}, {IN=>"\xC3\xA9 b\n"},
+ {OUT=>""},
+ {ENV => "LC_ALL=$mb_locale"}],
+ ['i-utf8-sharp-s', '-i',
+ {IN=>"\xC3\x9F a\n"}, {IN=>"SS b\n"},
+ {OUT=>""},
+ {ENV => "LC_ALL=$mb_locale"}];
}
@Tests = triple_test \@Tests;
diff --git a/tests/uniq/uniq.pl b/tests/uniq/uniq.pl
index 0df7ec62d..0f9be97ff 100755
--- a/tests/uniq/uniq.pl
+++ b/tests/uniq/uniq.pl
@@ -302,7 +302,18 @@ if ($mb_locale ne 'C')
my @Locale_Tests =
(
['w1-mb', '-w1', {IN => $trouble_with_w1}, {OUT => $trouble_with_w1},
- {ENV => "LC_ALL=$mb_locale"}]
+ {ENV => "LC_ALL=$mb_locale"}],
+ # -i with UTF-8 letters: memcasecmp is byte-wise and misses these.
+ # É/é fold together; ß/SS do not (not full Unicode case mapping).
+ ['i-utf8', '-i',
+ {IN=>"\xC3\x89\n\xC3\xA9\n"}, {OUT=>"\xC3\x89\n"},
+ {ENV => "LC_ALL=$mb_locale"}],
+ ['i-utf8-distinct', '-i',
+ {IN=>"\xC3\x89\nB\n"}, {OUT=>"\xC3\x89\nB\n"},
+ {ENV => "LC_ALL=$mb_locale"}],
+ ['i-utf8-sharp-s', '-i',
+ {IN=>"\xC3\x9F\nSS\n"}, {OUT=>"\xC3\x9F\nSS\n"},
+ {ENV => "LC_ALL=$mb_locale"}]
);
push @Tests, @Locale_Tests;
}
--
2.43.0