system/freebsd-src
system/freebsd-src
1
0
Fork 0
mirror of https://github.com/freebsd/freebsd-src synced 2024-09-20 00:33:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

libc/qsort: Don't allow interposing recursive calls

Browse source 
This causes problems when using ASAN with a runtime older than 12.0 since
the intercept does not expect qsort() to call itself using an interposable
function call. This results in infinite recursion and stack exhaustion
when a binary compiled with -fsanitize=address calls qsort.
See also https://bugs.llvm.org/show_bug.cgi?id=46832 and
https://reviews.llvm.org/D84509 (ASAN runtime patch).

To prevent this problem, this patch uses a static helper function
for the actual qsort() implementation. This prevents interposition and
allows for direct calls. As a nice side-effect, we can also move the
qsort_s checks to the top-level function and out of the recursive calls.

Reviewed By:	kib
Differential Revision: https://reviews.freebsd.org/D28133
This commit is contained in:
Alex Richardson 2021-02-18 10:12:29 +00:00
parent 2aa3ef285a
commit cbcfe28f9d
1 changed files with 50 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

102
lib/libc/stdlib/qsort.c
View file

@ -91,41 +91,23 @@ __unused
:(CMP(thunk, b, c) > 0 ? b : (CMP(thunk, a, c) < 0 ? a : c ));
}
/*
* The actual qsort() implementation is static to avoid preemptible calls when
* recursing. Also give them different names for improved debugging.
*/
#if defined(I_AM_QSORT_R)
void
qsort_r(void *a, size_t n, size_t es, void *thunk, cmp_t *cmp)
#define local_qsort local_qsort_r
#elif defined(I_AM_QSORT_S)
errno_t
qsort_s(void *a, rsize_t n, rsize_t es, cmp_t *cmp, void *thunk)
#else
#define thunk NULL
void
qsort(void *a, size_t n, size_t es, cmp_t *cmp)
#define local_qsort local_qsort_s
#endif
static void
local_qsort(void *a, size_t n, size_t es, cmp_t *cmp, void *thunk)
{
char *pa, *pb, *pc, *pd, *pl, *pm, *pn;
size_t d1, d2;
int cmp_result;
int swap_cnt;
#ifdef I_AM_QSORT_S
if (n > RSIZE_MAX) {
__throw_constraint_handler_s("qsort_s : n > RSIZE_MAX", EINVAL);
return (EINVAL);
} else if (es > RSIZE_MAX) {
__throw_constraint_handler_s("qsort_s : es > RSIZE_MAX", EINVAL);
return (EINVAL);
} else if (n != 0) {
if (a == NULL) {
__throw_constraint_handler_s("qsort_s : a == NULL", EINVAL);
return (EINVAL);
} else if (cmp == NULL) {
__throw_constraint_handler_s("qsort_s : cmp == NULL", EINVAL);
return (EINVAL);
}
}
#endif
loop:
swap_cnt = 0;
if (n < 7) {
@ -134,11 +116,7 @@ qsort(void *a, size_t n, size_t es, cmp_t *cmp)
pl > (char *)a && CMP(thunk, pl - es, pl) > 0;
pl -= es)
swapfunc(pl, pl - es, es);
#ifdef I_AM_QSORT_S
return (0);
#else
return;
#endif
}
pm = (char *)a + (n / 2) * es;
if (n > 7) {
@ -187,11 +165,7 @@ qsort(void *a, size_t n, size_t es, cmp_t *cmp)
pl > (char *)a && CMP(thunk, pl - es, pl) > 0;
pl -= es)
swapfunc(pl, pl - es, es);
#ifdef I_AM_QSORT_S
return (0);
#else
return;
#endif
}
pn = (char *)a + n * es;
@ -205,13 +179,7 @@ qsort(void *a, size_t n, size_t es, cmp_t *cmp)
if (d1 <= d2) {
/* Recurse on left partition, then iterate on right partition */
if (d1 > es) {
#if defined(I_AM_QSORT_R)
qsort_r(a, d1 / es, es, thunk, cmp);
#elif defined(I_AM_QSORT_S)
qsort_s(a, d1 / es, es, cmp, thunk);
#else
qsort(a, d1 / es, es, cmp);
#endif
local_qsort(a, d1 / es, es, cmp, thunk);
}
if (d2 > es) {
/* Iterate rather than recurse to save stack space */
@ -223,13 +191,7 @@ qsort(void *a, size_t n, size_t es, cmp_t *cmp)
} else {
/* Recurse on right partition, then iterate on left partition */
if (d2 > es) {
#if defined(I_AM_QSORT_R)
qsort_r(pn - d2, d2 / es, es, thunk, cmp);
#elif defined(I_AM_QSORT_S)
qsort_s(pn - d2, d2 / es, es, cmp, thunk);
#else
qsort(pn - d2, d2 / es, es, cmp);
#endif
local_qsort(pn - d2, d2 / es, es, cmp, thunk);
}
if (d1 > es) {
/* Iterate rather than recurse to save stack space */
@ -238,8 +200,44 @@ qsort(void *a, size_t n, size_t es, cmp_t *cmp)
goto loop;
}
}
#ifdef I_AM_QSORT_S
return (0);
#endif
}
#if defined(I_AM_QSORT_R)
void
qsort_r(void *a, size_t n, size_t es, void *thunk, cmp_t *cmp)
{
local_qsort_r(a, n, es, cmp, thunk);
}
#elif defined(I_AM_QSORT_S)
errno_t
qsort_s(void *a, rsize_t n, rsize_t es, cmp_t *cmp, void *thunk)
{
if (n > RSIZE_MAX) {
__throw_constraint_handler_s("qsort_s : n > RSIZE_MAX", EINVAL);
return (EINVAL);
} else if (es > RSIZE_MAX) {
__throw_constraint_handler_s("qsort_s : es > RSIZE_MAX",
EINVAL);
return (EINVAL);
} else if (n != 0) {
if (a == NULL) {
__throw_constraint_handler_s("qsort_s : a == NULL",
EINVAL);
return (EINVAL);
} else if (cmp == NULL) {
__throw_constraint_handler_s("qsort_s : cmp == NULL",
EINVAL);
return (EINVAL);
}
}
local_qsort_s(a, n, es, cmp, thunk);
return (0);
}
#else
void
qsort(void *a, size_t n, size_t es, cmp_t *cmp)
{
local_qsort(a, n, es, cmp, NULL);
}
#endif