We sporadically hit a segfault across different kinds of hardware on
Bash version 5.1.8(1)-release (x86_64-redhat-linux-gnu).
We found that run_sigchld_trap was run despite our bash script never
explicitly setting a SIGCHLD trap. Indeed, the trap_list array is empty
save for a trap set for EXIT at trap_list[0]. I am able to reproduce a
similar segfault when manually setting catch_flag=1 and
pending_traps[17]=1 in a gdb session. However, I cannot normally
reproduce the bug, and can only wait for it to show up about once a
month across thousands of servers that each run this script every few
minutes. We're currently suspecting that this is not a case of memory
corruption not just because we saw this across different hardware, but
every time the script crashes, we get this same stack trace.
Here's the stack trace and the relevant function in our bash code that
occasionally leads to this segfault:
#0 0x00007f9d3d42976b in kill () at ../sysdeps/unix/syscall-template.S:120
#1 0x000055b6eaa5e920 in termsig_handler (sig=11)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/sig.c:617
#2 0x000055b6eaa5eb21 in termsig_handler (sig=<optimized out>)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/sig.c:484
#3 termsig_sighandler (sig=<optimized out>) at
/usr/src/debug/bash-5.1.8-6.el9_1.x86_64/sig.c:539
#4 <signal handler called>
#5 __strlen_avx2_rtm () at ../sysdeps/x86_64/multiarch/strlen-avx2.S:74
#6 0x000055b6eaa3d554 in run_sigchld_trap (nchild=1)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/jobs.c:4205
#7 0x000055b6eaa5a4e7 in run_pending_traps () at
/usr/src/debug/bash-5.1.8-6.el9_1.x86_64/trap.c:370
#8 0x000055b6eaa22060 in execute_command_internal (command=<optimized
out>,
asynchronous=<optimized out>, pipe_in=<optimized out>,
pipe_out=<optimized out>,
fds_to_close=<optimized out>) at
/usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:1141
#9 0x000055b6eaa233db in execute_connection
(fds_to_close=0x55b6eb1940b0, pipe_out=-1, pipe_in=-1,
asynchronous=0, command=0x55b6eb2611b0)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:2727
pipe_out=-1, fds_to_close=0x55b6eb1940b0)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:1032
#11 0x000055b6eaa24758 in execute_command (command=0x55b6eb2611b0)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:399
#12 0x000055b6eaa2339e in execute_connection
(fds_to_close=0x55b6eb193ea0, pipe_out=-1, pipe_in=-1,
asynchronous=0, command=0x55b6eb266390)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:2718
pipe_out=-1, fds_to_close=0x55b6eb193ea0)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:1032
#14 0x000055b6eaa24758 in execute_command (command=0x55b6eb266390)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:399
#15 0x000055b6eaa2339e in execute_connection
(fds_to_close=0x55b6eb268840, pipe_out=-1, pipe_in=-1,
asynchronous=0, command=0x55b6eb266330)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:2718
pipe_out=-1, fds_to_close=0x55b6eb268840)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:1032
#17 0x000055b6eaa24758 in execute_command (command=0x55b6eb266330)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:399
#18 0x000055b6eaa2339e in execute_connection
(fds_to_close=0x55b6eb193e50, pipe_out=-1, pipe_in=-1,
asynchronous=0, command=0x55b6eb266730)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:2718
pipe_out=-1, fds_to_close=0x55b6eb193e50)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:1032
#20 0x000055b6eaa24758 in execute_command (command=0x55b6eb266730)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:399
#21 0x000055b6eaa2339e in execute_connection
(fds_to_close=0x55b6eb193d70, pipe_out=-1, pipe_in=-1,
asynchronous=0, command=0x55b6eb1e1480)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:2718
pipe_out=-1, fds_to_close=0x55b6eb193d70)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:1032
#23 0x000055b6eaa24758 in execute_command (command=0x55b6eb1e1480)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:399
#24 0x000055b6eaa2339e in execute_connection
(fds_to_close=0x55b6eb24c240, pipe_out=-1, pipe_in=-1,
asynchronous=0, command=0x55b6eb1e1420)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:2718
pipe_out=-1, fds_to_close=0x55b6eb24c240)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:1032
#26 0x000055b6eaa24e4b in execute_in_subshell
(command=command@entry=0x55b6eb256f40,
asynchronous=asynchronous@entry=0, pipe_in=pipe_in@entry=-1,
pipe_out=pipe_out@entry=-1,
fds_to_close=fds_to_close@entry=0x55b6eb24c240)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:1697
#27 0x000055b6eaa21cb4 in execute_command_internal (command=0x55b6eb256f40,
pipe_out=-1, fds_to_close=0x55b6eb24c240)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:1032
#30 0x000055b6eaa22193 in execute_command_internal (command=0x55b6eb26a790,
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:1024
#36 0x000055b6eaa2ada0 in execute_function (var=var@entry=0x55b6eb1e7a10,
words=words@entry=0x55b6eb259e40, flags=flags@entry=72,
pipe_out=-1, fds_to_close=0x55b6eb25b160)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:1032
#43 0x000055b6eaa24758 in execute_command (command=0x55b6eb263570)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:399
#44 0x000055b6eaa22cae in execute_if_command (if_command=<optimized out>)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:3709
pipe_out=-1, fds_to_close=0x55b6eb2573c0)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:1032
#48 0x000055b6eaa22193 in execute_command_internal (command=0x55b6eb25ba90,
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:1024
#49 0x000055b6eaa2ada0 in execute_function (var=var@entry=0x55b6eb201a80,
words=words@entry=0x55b6eb274fc0, flags=flags@entry=64,
pipe_out=-1, fds_to_close=0x55b6eb2573c0)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:1032
#58 0x000055b6eaa24758 in execute_command (command=0x55b6eb24f840)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:399
#59 0x000055b6eaa224b0 in execute_case_command (case_command=<optimized
out>)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:3588
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:966
#61 0x000055b6eaa24758 in execute_command (command=0x55b6eb257260)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/execute_cmd.c:399
#62 0x000055b6eaa160e9 in reader_loop () at
/usr/src/debug/bash-5.1.8-6.el9_1.x86_64/eval.c:171
#63 0x000055b6eaa0795e in main (argc=4, argv=0x7ffdf47ebd68,
env=0x7ffdf47ebd90)
at /usr/src/debug/bash-5.1.8-6.el9_1.x86_64/shell.c:821
function validateHardwareCached()
{
)
fi
) {flockFD}< $(dirname $validateHardwareCacheFile)
)
}
At the time of the segfault, line_number corresponded to the
third-to-last line, the one that reads ") {flockFD}< $(dirname
$validateHardwareCacheFile)".
Any insight into what may be causing this would be much appreciated!