build fails because a bug in lib/unicore/mktables occurs

[hilti2]

This is a bug report for perl from hilko.meyer@gmx.de,
generated with the help of perlbug 1.42 running under perl 5.36.3.

---

[Please describe your issue here]

The build fails in the end with the following error:

LD_LIBRARY_PATH=/usr/ports/lang/perl5.36/work/perl-5.36.3 ./miniperl -Ilib make_ext.pl lib/auto/Unicode/Normalize/Normalize.so MAKE="/usr/bin/make" LIBPERL_A=libperl.so.5.36.3 LINKTYPE=dynamic
Making header files for XS...
Use of uninitialized value $tab[2] in substitution (s///) at ./mkheader line 122.
Use of uninitialized value $_[0] in pattern match (m//) at ./mkheader line 108.
Illegal hexadecimal digit '!' ignored at ./mkheader line 124.
Use of uninitialized value $tab[1] in string eq at ./mkheader line 125.
Weird Canonical Decomposition of U+0000 at ./mkheader line 155.
require ./mkheader called at Makefile.PL line 13
Unsuccessful Makefile.PL(dist/Unicode-Normalize): code=65280 at make_ext.pl line 532.
*** Error code 2

But much earlier many, many bugs in lib/unicore/mktables are reported:
LD_LIBRARY_PATH=/usr/ports/lang/perl5.36/work/perl-5.36.3 ./miniperl -Ilib lib/unicore/mktables -C lib/unicore -P pod -maketest -makelist -p

lib/unicore/mktables: Bug in lib/unicore/mktables. Please report it by
running perlbug or if that is unavailable, by sending email to
perbug@perl.org: Unrecognized parameters 'Required_Even_in_Debug_Skip =>
1' to new() for Input_file object for PropertyAliases.txt. Skipped; at lib/unicore/mktables line 19533.
Use of uninitialized value in concatenation (.) or string at lib/unicore/mktables line 2640.

lib/unicore/mktables: Bug in lib/unicore/mktables. Please report it by
running perlbug or if that is unavailable, by sending email to
perbug@perl.org: Unrecognized parameters 'Progress_Message => Finishing
property setup' to new() for Input_file object for . Skipped; at lib/unicore/mktables line 19533.
Use of uninitialized value $progress in concatenation (.) or string at lib/unicore/mktables line 2510.

The complete build log is attached.

[Please do not change anything below this line]

---

Flags:
category=core
severity=high

Site configuration information for perl 5.36.3:

Configured by mat at Wed Nov 29 17:10:26 CET 2023.

Summary of my perl5 (revision 5 version 36 subversion 3) configuration:

Platform:
osname=freebsd
osvers=13.2-release-p3
archname=amd64-freebsd-thread-multi
uname='freebsd falko.fritz.box 13.2-release-p3 freebsd 13.2-release-p3 releng13.2-a1c915cc7 falko amd64 '
config_args='-Accflags=-DUSE_THREAD_SAFE_LOCALE -Darchlib=/usr/local/lib/perl5/5.36/mach -Dcc=cc -Dcf_by=mat -Dcf_email=mat@FreeBSD.org -Dcf_time=Wed Nov 29 17:10:26 CET 2023 -Dinc_version_list=none -Dlibperl=libperl.so.5.36.3 -Dman1dir=/usr/local/lib/perl5/5.36/perl/man/man1 -Dman3dir=/usr/local/lib/perl5/5.36/perl/man/man3 -Dprefix=/usr/local -Dprivlib=/usr/local/lib/perl5/5.36 -Dscriptdir=/usr/local/bin -Dsitearch=/usr/local/lib/perl5/site_perl/mach/5.36 -Dsitelib=/usr/local/lib/perl5/site_perl -Dsiteman1dir=/usr/local/lib/perl5/site_perl/man/man1 -Dsiteman3dir=/usr/local/lib/perl5/site_perl/man/man3 -Dusenm=n -Duseshrplib -sde -Ui_iconv -Ui_malloc -Uinstallusrbinperl -Alddlflags=-L/usr/ports/lang/perl5.36/work/perl-5.36.3 -L/usr/local/lib/perl5/5.36/mach/CORE -lperl
-Dshrpldflags=$(LDDLFLAGS:N-L/usr/ports/lang/perl5.36/work/perl-5.36.3:N-L/usr/local/lib/perl5/5.36/mach/CORE:N-lperl) -Wl,-soname,$(LIBPERL:R) -Doptimize=-O2 -pipe -march=amdfam10 -fstack-protector-strong -fno-strict-aliasing -Dusedtrace -Ui_gdbm -Dusemultiplicity=y -Duse64bitint -Dusemymalloc=n -Dusethreads=y'
hint=recommended
useposix=true
d_sigaction=define
useithreads=define
usemultiplicity=define
use64bitint=define
use64bitall=define
uselongdouble=undef
usemymalloc=n
default_inc_excludes_dot=define
Compiler:
cc='cc'
ccflags ='-DHAS_FPSETMASK -DHAS_FLOATINGPOINT_H -DUSE_THREAD_SAFE_LOCALE -fno-strict-aliasing -pipe -fstack-protector-strong -I/usr/local/include'
optimize='-O2 -pipe -march=amdfam10 -fstack-protector-strong -fno-strict-aliasing '
cppflags='-DHAS_FPSETMASK -DHAS_FLOATINGPOINT_H -DUSE_THREAD_SAFE_LOCALE -fno-strict-aliasing -pipe -fstack-protector-strong -I/usr/local/include'
ccversion=''
gccversion='FreeBSD Clang 14.0.5 (https://github.com/llvm/llvm-project.git llvmorg-14.0.5-0-gc12386ae247c)'
gccosandvers=''
intsize=4
longsize=8
ptrsize=8
doublesize=8
byteorder=12345678
doublekind=3
d_longlong=define
longlongsize=8
d_longdbl=define
longdblsize=16
longdblkind=3
ivtype='long'
ivsize=8
nvtype='double'
nvsize=8
Off_t='off_t'
lseeksize=8
alignbytes=8
prototype=define
Linker and Libraries:
ld='cc'
ldflags ='-pthread -Wl,-E -fstack-protector-strong -L/usr/local/lib'
libpth=/usr/lib /usr/local/lib /usr/lib/clang/14.0.5/lib
libs=-lgdbm -ldl -lm -lcrypt -lutil
perllibs=-ldl -lm -lcrypt -lutil
libc=
so=so
useshrplib=true
libperl=libperl.so.5.36.3
gnulibc_version=''
Dynamic Linking:
dlsrc=dl_dlopen.xs
dlext=so
d_dlsymun=undef
ccdlflags=' -Wl,-R/usr/local/lib/perl5/5.36/mach/CORE'
cccdlflags='-DPIC -fPIC'
lddlflags='-shared -L/usr/local/lib/perl5/5.36/mach/CORE -lperl -L/usr/local/lib -fstack-protector-strong'

---

@inc for perl 5.36.3:
/usr/local/lib/perl5/site_perl/mach/5.36
/usr/local/lib/perl5/site_perl
/usr/local/lib/perl5/5.36/mach
/usr/local/lib/perl5/5.36

---

Environment for perl 5.36.3:
HOME=/root
LANG=C.UTF-8
LANGUAGE (unset)
LD_LIBRARY_PATH (unset)
LOGDIR (unset)
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/root/bin
PERL_BADLANG (unset)
SHELL=/bin/csh
perlcompilelog.txt

[jkeenan]

In the directory where you were attempting to compile this perl, can you:

1. Execute uname -a
2. Provide the way you configured, i.e., report back how you called something like this:

$ sh ./Configure -des -Dusedevel -Duseithreads

... and report back the results of each? (I'm assuming that what you supplied for site configuration item was your vendor perl.) Thanks.

[jkeenan]

Also, if you have access to a git checkout of the Perl 5 core distribution, could you say git checkout v5.36.3, then configure and build with the same command-line switches you used above and report back whether or not you encountered the same errors? Thanks.

[hilti2]

1. uname -a:
   FreeBSD falko.fritz.box 13.3-RELEASE FreeBSD 13.3-RELEASE releng/13.3-n257428-80d2b634ddf0 FALKO amd64

2. To give more context. I use the FreeBSD-Ports-Framework to compile my software. The supplied configuration is for a perl 5.36.3 compiled on FreeBSD 13.2. I recently updated to FreeBSD 13.3 and wanted to recompile perl with the same options, but it didn't worked anymore. The FreeBSD perl maintainer Mathieu Arnold asked me to file the bugreport here.

[hilti2]

I got it building now and now I'm puzzled. Just to be sure I tried it with an empty make.conf and it worked. Then I looked what options were set there and its just CPUTYPE?= amdfam10.

With FreeBSD 13.3 LLVM and the clang compiler have been upgraded from 14.0.5 to version 17.0.6. No idea whats hapening here. Maybe a regression in clang 17.0.6 with the -march=amdfam10 option.

Just for the record a diff of the Site configuration information from perlbug:

--- perlbug.rep	2024-04-10 14:34:56.477978354 +0200
+++ perlbug.rep-working.txt	2024-04-10 20:39:59.995610000 +0200
 ---
 Site configuration information for perl 5.36.3:
 
@@ -52,11 +30,11 @@
    
   Platform:
     osname=freebsd
-    osvers=13.2-release-p3
+    osvers=13.3-release
     archname=amd64-freebsd-thread-multi
-    uname='freebsd falko.fritz.box 13.2-release-p3 freebsd 13.2-release-p3 releng13.2-a1c915cc7 falko amd64 '
+    uname='freebsd falko.fritz.box 13.3-release freebsd 13.3-release releng13.3-n257428-80d2b634ddf0 falko amd64 '
     config_args='-Accflags=-DUSE_THREAD_SAFE_LOCALE -Darchlib=/usr/local/lib/perl5/5.36/mach -Dcc=cc -Dcf_by=mat -Dcf_email=mat@FreeBSD.org -Dcf_time=Wed Nov 29 17:10:26 CET 2023 -Dinc_version_list=none -Dlibperl=libperl.so.5.36.3 -Dman1dir=/usr/local/lib/perl5/5.36/perl/man/man1 -Dman3dir=/usr/local/lib/perl5/5.36/perl/man/man3 -Dprefix=/usr/local -Dprivlib=/usr/local/lib/perl5/5.36 -Dscriptdir=/usr/local/bin -Dsitearch=/usr/local/lib/perl5/site_perl/mach/5.36 -Dsitelib=/usr/local/lib/perl5/site_perl -Dsiteman1dir=/usr/local/lib/perl5/site_perl/man/man1 -Dsiteman3dir=/usr/local/lib/perl5/site_perl/man/man3 -Dusenm=n -Duseshrplib -sde -Ui_iconv -Ui_malloc -Uinstallusrbinperl -Alddlflags=-L/usr/ports/lang/perl5.36/work/perl-5.36.3 -L/usr/local/lib/perl5/5.36/mach/CORE -lperl
--Dshrpldflags=$(LDDLFLAGS:N-L/usr/ports/lang/perl5.36/work/perl-5.36.3:N-L/usr/local/lib/perl5/5.36/mach/CORE:N-lperl) -Wl,-soname,$(LIBPERL:R) -Doptimize=-O2 -pipe -march=amdfam10  -fstack-protector-strong -fno-strict-aliasing  -Dusedtrace -Ui_gdbm -Dusemultiplicity=y -Duse64bitint -Dusemymalloc=n -Dusethreads=y'
+-Dshrpldflags=$(LDDLFLAGS:N-L/usr/ports/lang/perl5.36/work/perl-5.36.3:N-L/usr/local/lib/perl5/5.36/mach/CORE:N-lperl) -Wl,-soname,$(LIBPERL:R) -Doptimize=-O2 -pipe  -fstack-protector-strong -fno-strict-aliasing  -Dusedtrace -Ui_gdbm -Dusemultiplicity=y -Duse64bitint -Dusemymalloc=n -Dusethreads=y'
     hint=recommended
     useposix=true
     d_sigaction=define
@@ -70,10 +48,10 @@
   Compiler:
     cc='cc'
     ccflags ='-DHAS_FPSETMASK -DHAS_FLOATINGPOINT_H -DUSE_THREAD_SAFE_LOCALE -fno-strict-aliasing -pipe -fstack-protector-strong -I/usr/local/include'
-    optimize='-O2 -pipe -march=amdfam10 -fstack-protector-strong -fno-strict-aliasing '
+    optimize='-O2 -pipe -fstack-protector-strong -fno-strict-aliasing '
     cppflags='-DHAS_FPSETMASK -DHAS_FLOATINGPOINT_H -DUSE_THREAD_SAFE_LOCALE -fno-strict-aliasing -pipe -fstack-protector-strong -I/usr/local/include'
     ccversion=''
-    gccversion='FreeBSD Clang 14.0.5 (https://github.com/llvm/llvm-project.git llvmorg-14.0.5-0-gc12386ae247c)'
+    gccversion='FreeBSD Clang 17.0.6 (https://github.com/llvm/llvm-project.git llvmorg-17.0.6-0-g6009708b4367)'
     gccosandvers=''
     intsize=4
     longsize=8
@@ -97,7 +75,7 @@
   Linker and Libraries:
     ld='cc'
     ldflags ='-pthread -Wl,-E  -fstack-protector-strong -L/usr/local/lib'
-    libpth=/usr/lib /usr/local/lib /usr/lib/clang/14.0.5/lib
+    libpth=/usr/lib /usr/local/lib /usr/lib/clang/17/lib
     libs=-lgdbm -ldl -lm -lcrypt -lutil
     perllibs=-ldl -lm -lcrypt -lutil
     libc=

[tonycoz]

I'm confused, you said here removing -march=amdfam10 allowed it to build, but you've reported a similar bug on the list (which also appears to use the system perl -V output rather than the perl being built).

[hilti2]

First of all sorry for the double report. The mail to perl-porters was the first try to but I got a bounce and thought the mail didn't got through and it would be less hassle to report it here.

So yes, I blindly ran perlbug without realizing it would run the version I wanted to replace. But to my defense, the build fails before it builds perlbug. :-)

So the perlbug reports show the output of a successfull build.

To summarize it:

working build:
FreeBSD 13.2 with LLVM/clang 14.0.5 with -march=amdfam10
FreeBSD 13.3 with LLVM/clang 17.0.6 without -march=amdfam10

not working build:
FreeBSD 13.3 with LLVM/clang 17.0.6 with -march=amdfam10

[tonycoz]

I've reproduced this in blead on a VM with a Ryzen 7 in FreeBSD 13.3 release.

[tonycoz]

I see similar behaviour with clang 17.0.6 with -march=amdfam10 on Linux too, Debian stable, built with:

./Configure -des -Dusedevel -Dcc=clang-17 -Accflags=-march=amdfam10

on AMD EPYC 7713 64-Core Processor (linode if it matters.)

The generated code produces SIGILL on Intel(R) Core(TM) i7-10700F CPU @ 2.90GHz, on an EXTRQ instruction, which isn't available on Intel.

I see the reported failure with Debian clang version 18.1.4 (++20240410110204+b6ebea7972cd-1~exp1~20240410230232.97) (from apt.llvm.org) on the AMD EPYC above.

[tonycoz]

So far I suspect a bug in auto vectorization reflected in the output of lc():

~/dev/perl/git/perl$ ./miniperl -le '$x = "Requiredd_Even_in"; print lc $x' | hd
00000000  72 65 71 75 69 72 65 64  84 5f 45 76 65 6e 5f 69  |required._Even_i|
00000010  6e 0a                                             |n.|
00000012
~/dev/perl/git/perl$ ./miniperl -le '$x = "Required_Even_in"; print lc $x' | hd
00000000  72 65 71 75 69 72 65 64  7f 45 76 65 6e 5f 69 6e  |required.Even_in|
00000010  0a                                                |.|
00000011

Note how the lower case of the character at offset 8 is being corrupted, this does not happen in the string is 15 characters or less, i assume because the vectorization doesn't perform a second loop.

Adding enough data for a third loop doesn't cause any further problem.

I plan to make a simple reproducer to report upstream.

[tonycoz]

Worked up a simpler reproducer, which has been reported llvm/llvm-project#89008

$ cat 22132.c
#include <stdio.h>
#include <string.h>


typedef unsigned char U8;
typedef unsigned short U16;
typedef unsigned int U32;
typedef unsigned long long WIDEST_UTYPE;

#     define isUPPER_A(c)  inRANGE(c, 'A', 'Z')
#define isUPPER(c)   isUPPER_A(c)
#define toLOWER(c)  (isUPPER(c) ? (U8)((c) + ('a' - 'A')) : (c))
#define inRANGE(c, l, u) (   \
   (  (sizeof(c) == sizeof(U8))  ? inRANGE_helper_(U8, (c), (l), ((u)))     \
    : (sizeof(c) == sizeof(U16)) ? inRANGE_helper_(U16,(c), (l), ((u)))     \
    : (sizeof(c) == sizeof(U32)) ? inRANGE_helper_(U32,(c), (l), ((u)))     \
             : ( inRANGE_helper_(WIDEST_UTYPE,(c), (l), ((u))))))

#define inRANGE_helper_(cast, c, l, u)                                      \
                    withinCOUNT_KNOWN_VALID_(((cast) (c)), (l), ((u) - (l)))
#define withinCOUNT_KNOWN_VALID_(c, l, n)                                   \
    ((((WIDEST_UTYPE) (c)) - ASSERT_NOT_PTR(l))                             \
                                   <= ((WIDEST_UTYPE) ASSERT_NOT_PTR(n)))

#  define ASSERT_NOT_PTR(x) (x)

void lower_s(const char *s, const char *send, char *d) {
  for (; s < send; ++s, ++d)
    *d = toLOWER(*s);
  *d = 0;
}

int main() {
  char str[] = "Required_Even_ix";
  char out[sizeof(str)];

  lower_s(str, str+strlen(str), out);

  puts(out);
}
$ clang-17 -O2 -march=amdfam10 22132.c && ./a.out | hd
00000000  72 65 71 75 69 72 65 64  7f 45 76 65 6e 5f 69 78  |required.Even_ix|
00000010  0a                                                |.|
00000011
$ clang-14 -O2 -march=amdfam10 22132.c && ./a.out | hd
00000000  72 65 71 75 69 72 65 64  5f 65 76 65 6e 5f 69 78  |required_even_ix|
00000010  0a                                                |.|
00000011