Skip to content

Commit dad74bd

Browse files
jasonejasone
committed
Convert witness_assert_lockless() to witness_assert_lock_depth().
This makes it possible to make lock state assertions about precisely which locks are held.
1 parent c6943ac commit dad74bd

File tree

5 files changed

+102
-81
lines changed

5 files changed

+102
-81
lines changed

‎include/jemalloc/internal/private_symbols.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -612,14 +612,14 @@ valgrind_freelike_block
612612
valgrind_make_mem_defined
613613
valgrind_make_mem_noaccess
614614
valgrind_make_mem_undefined
615-
witness_assert_lockless
615+
witness_assert_lock_depth
616616
witness_assert_not_owner
617617
witness_assert_owner
618618
witness_fork_cleanup
619619
witness_init
620620
witness_lock
621621
witness_lock_error
622-
witness_lockless_error
622+
witness_lock_depth_error
623623
witness_not_owner_error
624624
witness_owner
625625
witness_owner_error

‎include/jemalloc/internal/witness.h‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -91,10 +91,12 @@ extern witness_not_owner_error_t *witness_not_owner_error;
9191
void witness_not_owner_error(const witness_t *witness);
9292
#endif
9393
#ifdef JEMALLOC_JET
94-
typedef void (witness_lockless_error_t)(const witness_list_t *);
95-
extern witness_lockless_error_t *witness_lockless_error;
94+
typedef void (witness_lock_depth_error_t)(const witness_list_t *,
95+
unsigned depth);
96+
extern witness_lock_depth_error_t *witness_lock_depth_error;
9697
#else
97-
void witness_lockless_error(const witness_list_t *witnesses);
98+
void witness_lock_depth_error(const witness_list_t *witnesses,
99+
unsigned depth);
98100
#endif
99101

100102
void witnesses_cleanup(tsd_t *tsd);
@@ -111,7 +113,7 @@ void witness_postfork_child(tsd_t *tsd);
111113
bool witness_owner(tsd_t *tsd, const witness_t *witness);
112114
void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
113115
void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
114-
void witness_assert_lockless(tsdn_t *tsdn);
116+
void witness_assert_lock_depth(tsdn_t *tsdn, unsigned depth);
115117
void witness_lock(tsdn_t *tsdn, witness_t *witness);
116118
void witness_unlock(tsdn_t *tsdn, witness_t *witness);
117119
#endif
@@ -175,9 +177,10 @@ witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness)
175177
}
176178

177179
JEMALLOC_INLINE void
178-
witness_assert_lockless(tsdn_t *tsdn)
180+
witness_assert_lock_depth(tsdn_t *tsdn, unsigned depth)
179181
{
180182
tsd_t *tsd;
183+
unsigned d;
181184
witness_list_t *witnesses;
182185
witness_t *w;
183186

@@ -188,10 +191,16 @@ witness_assert_lockless(tsdn_t *tsdn)
188191
return;
189192
tsd = tsdn_tsd(tsdn);
190193

194+
d = 0;
191195
witnesses = tsd_witnessesp_get(tsd);
192196
w = ql_last(witnesses, link);
193-
if (w != NULL)
194-
witness_lockless_error(witnesses);
197+
if (w != NULL) {
198+
ql_foreach(w, witnesses, link) {
199+
d++;
200+
}
201+
}
202+
if (d != depth)
203+
witness_lock_depth_error(witnesses, depth);
195204
}
196205

197206
JEMALLOC_INLINE void

‎src/jemalloc.c‎

Lines changed: 37 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1581,7 +1581,7 @@ ialloc_body(size_t size, bool zero, tsdn_t **tsdn, size_t *usize,
15811581

15821582
tsd = tsd_fetch();
15831583
*tsdn = tsd_tsdn(tsd);
1584-
witness_assert_lockless(tsd_tsdn(tsd));
1584+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
15851585

15861586
ind = size2index(size);
15871587
if (unlikely(ind >= NSIZES))
@@ -1619,7 +1619,7 @@ ialloc_post_check(void *ret, tsdn_t *tsdn, size_t usize, const char *func,
16191619
assert(usize == isalloc(tsdn, ret, config_prof));
16201620
*tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize;
16211621
}
1622-
witness_assert_lockless(tsdn);
1622+
witness_assert_lock_depth(tsdn, 0);
16231623
}
16241624

16251625
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
@@ -1704,7 +1704,7 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
17041704
goto label_oom;
17051705
}
17061706
tsd = tsd_fetch();
1707-
witness_assert_lockless(tsd_tsdn(tsd));
1707+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
17081708
if (size == 0)
17091709
size = 1;
17101710

@@ -1745,7 +1745,7 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
17451745
UTRACE(0, size, result);
17461746
JEMALLOC_VALGRIND_MALLOC(result != NULL, tsd_tsdn(tsd), result, usize,
17471747
false);
1748-
witness_assert_lockless(tsd_tsdn(tsd));
1748+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
17491749
return (ret);
17501750
label_oom:
17511751
assert(result == NULL);
@@ -1755,7 +1755,7 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
17551755
abort();
17561756
}
17571757
ret = ENOMEM;
1758-
witness_assert_lockless(tsd_tsdn(tsd));
1758+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
17591759
goto label_return;
17601760
}
17611761

@@ -1873,7 +1873,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
18731873
size_t usize;
18741874
UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
18751875

1876-
witness_assert_lockless(tsd_tsdn(tsd));
1876+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
18771877

18781878
assert(ptr != NULL);
18791879
assert(malloc_initialized() || IS_INITIALIZER);
@@ -1901,7 +1901,7 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path)
19011901
{
19021902
UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
19031903

1904-
witness_assert_lockless(tsd_tsdn(tsd));
1904+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
19051905

19061906
assert(ptr != NULL);
19071907
assert(malloc_initialized() || IS_INITIALIZER);
@@ -1947,7 +1947,7 @@ je_realloc(void *ptr, size_t size)
19471947
malloc_thread_init();
19481948
tsd = tsd_fetch();
19491949

1950-
witness_assert_lockless(tsd_tsdn(tsd));
1950+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
19511951

19521952
old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
19531953
if (config_valgrind && unlikely(in_valgrind)) {
@@ -1994,7 +1994,7 @@ je_realloc(void *ptr, size_t size)
19941994
UTRACE(ptr, size, ret);
19951995
JEMALLOC_VALGRIND_REALLOC(maybe, tsdn, ret, usize, maybe, ptr,
19961996
old_usize, old_rzsize, maybe, false);
1997-
witness_assert_lockless(tsdn);
1997+
witness_assert_lock_depth(tsdn, 0);
19981998
return (ret);
19991999
}
20002000

@@ -2005,12 +2005,12 @@ je_free(void *ptr)
20052005
UTRACE(ptr, 0, 0);
20062006
if (likely(ptr != NULL)) {
20072007
tsd_t *tsd = tsd_fetch();
2008-
witness_assert_lockless(tsd_tsdn(tsd));
2008+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
20092009
if (likely(!malloc_slow))
20102010
ifree(tsd, ptr, tcache_get(tsd, false), false);
20112011
else
20122012
ifree(tsd, ptr, tcache_get(tsd, false), true);
2013-
witness_assert_lockless(tsd_tsdn(tsd));
2013+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
20142014
}
20152015
}
20162016

@@ -2239,7 +2239,7 @@ imallocx_body(size_t size, int flags, tsdn_t **tsdn, size_t *usize,
22392239

22402240
tsd = tsd_fetch();
22412241
*tsdn = tsd_tsdn(tsd);
2242-
witness_assert_lockless(tsd_tsdn(tsd));
2242+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
22432243

22442244
if (likely(flags == 0)) {
22452245
szind_t ind = size2index(size);
@@ -2374,7 +2374,7 @@ je_rallocx(void *ptr, size_t size, int flags)
23742374
assert(malloc_initialized() || IS_INITIALIZER);
23752375
malloc_thread_init();
23762376
tsd = tsd_fetch();
2377-
witness_assert_lockless(tsd_tsdn(tsd));
2377+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
23782378

23792379
if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
23802380
unsigned arena_ind = MALLOCX_ARENA_GET(flags);
@@ -2421,15 +2421,15 @@ je_rallocx(void *ptr, size_t size, int flags)
24212421
UTRACE(ptr, size, p);
24222422
JEMALLOC_VALGRIND_REALLOC(maybe, tsd_tsdn(tsd), p, usize, no, ptr,
24232423
old_usize, old_rzsize, no, zero);
2424-
witness_assert_lockless(tsd_tsdn(tsd));
2424+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
24252425
return (p);
24262426
label_oom:
24272427
if (config_xmalloc && unlikely(opt_xmalloc)) {
24282428
malloc_write("<jemalloc>: Error in rallocx(): out of memory\n");
24292429
abort();
24302430
}
24312431
UTRACE(ptr, size, 0);
2432-
witness_assert_lockless(tsd_tsdn(tsd));
2432+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
24332433
return (NULL);
24342434
}
24352435

@@ -2525,7 +2525,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
25252525
assert(malloc_initialized() || IS_INITIALIZER);
25262526
malloc_thread_init();
25272527
tsd = tsd_fetch();
2528-
witness_assert_lockless(tsd_tsdn(tsd));
2528+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
25292529

25302530
old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
25312531

@@ -2566,7 +2566,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
25662566
old_usize, old_rzsize, no, zero);
25672567
label_not_resized:
25682568
UTRACE(ptr, size, ptr);
2569-
witness_assert_lockless(tsd_tsdn(tsd));
2569+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
25702570
return (usize);
25712571
}
25722572

@@ -2581,14 +2581,14 @@ je_sallocx(const void *ptr, int flags)
25812581
malloc_thread_init();
25822582

25832583
tsdn = tsdn_fetch();
2584-
witness_assert_lockless(tsdn);
2584+
witness_assert_lock_depth(tsdn, 0);
25852585

25862586
if (config_ivsalloc)
25872587
usize = ivsalloc(tsdn, ptr, config_prof);
25882588
else
25892589
usize = isalloc(tsdn, ptr, config_prof);
25902590

2591-
witness_assert_lockless(tsdn);
2591+
witness_assert_lock_depth(tsdn, 0);
25922592
return (usize);
25932593
}
25942594

@@ -2602,7 +2602,7 @@ je_dallocx(void *ptr, int flags)
26022602
assert(malloc_initialized() || IS_INITIALIZER);
26032603

26042604
tsd = tsd_fetch();
2605-
witness_assert_lockless(tsd_tsdn(tsd));
2605+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
26062606
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
26072607
if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
26082608
tcache = NULL;
@@ -2616,21 +2616,21 @@ je_dallocx(void *ptr, int flags)
26162616
ifree(tsd, ptr, tcache, false);
26172617
else
26182618
ifree(tsd, ptr, tcache, true);
2619-
witness_assert_lockless(tsd_tsdn(tsd));
2619+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
26202620
}
26212621

26222622
JEMALLOC_ALWAYS_INLINE_C size_t
26232623
inallocx(tsdn_t *tsdn, size_t size, int flags)
26242624
{
26252625
size_t usize;
26262626

2627-
witness_assert_lockless(tsdn);
2627+
witness_assert_lock_depth(tsdn, 0);
26282628

26292629
if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0))
26302630
usize = s2u(size);
26312631
else
26322632
usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags));
2633-
witness_assert_lockless(tsdn);
2633+
witness_assert_lock_depth(tsdn, 0);
26342634
return (usize);
26352635
}
26362636

@@ -2647,7 +2647,7 @@ je_sdallocx(void *ptr, size_t size, int flags)
26472647
usize = inallocx(tsd_tsdn(tsd), size, flags);
26482648
assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof));
26492649

2650-
witness_assert_lockless(tsd_tsdn(tsd));
2650+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
26512651
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
26522652
if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
26532653
tcache = NULL;
@@ -2661,7 +2661,7 @@ je_sdallocx(void *ptr, size_t size, int flags)
26612661
isfree(tsd, ptr, usize, tcache, false);
26622662
else
26632663
isfree(tsd, ptr, usize, tcache, true);
2664-
witness_assert_lockless(tsd_tsdn(tsd));
2664+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
26652665
}
26662666

26672667
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
@@ -2677,13 +2677,13 @@ je_nallocx(size_t size, int flags)
26772677
return (0);
26782678

26792679
tsdn = tsdn_fetch();
2680-
witness_assert_lockless(tsdn);
2680+
witness_assert_lock_depth(tsdn, 0);
26812681

26822682
usize = inallocx(tsdn, size, flags);
26832683
if (unlikely(usize > HUGE_MAXCLASS))
26842684
return (0);
26852685

2686-
witness_assert_lockless(tsdn);
2686+
witness_assert_lock_depth(tsdn, 0);
26872687
return (usize);
26882688
}
26892689

@@ -2698,9 +2698,9 @@ je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
26982698
return (EAGAIN);
26992699

27002700
tsd = tsd_fetch();
2701-
witness_assert_lockless(tsd_tsdn(tsd));
2701+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
27022702
ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen);
2703-
witness_assert_lockless(tsd_tsdn(tsd));
2703+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
27042704
return (ret);
27052705
}
27062706

@@ -2714,9 +2714,9 @@ je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp)
27142714
return (EAGAIN);
27152715

27162716
tsdn = tsdn_fetch();
2717-
witness_assert_lockless(tsdn);
2717+
witness_assert_lock_depth(tsdn, 0);
27182718
ret = ctl_nametomib(tsdn, name, mibp, miblenp);
2719-
witness_assert_lockless(tsdn);
2719+
witness_assert_lock_depth(tsdn, 0);
27202720
return (ret);
27212721
}
27222722

@@ -2731,9 +2731,9 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
27312731
return (EAGAIN);
27322732

27332733
tsd = tsd_fetch();
2734-
witness_assert_lockless(tsd_tsdn(tsd));
2734+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
27352735
ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
2736-
witness_assert_lockless(tsd_tsdn(tsd));
2736+
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
27372737
return (ret);
27382738
}
27392739

@@ -2744,9 +2744,9 @@ je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
27442744
tsdn_t *tsdn;
27452745

27462746
tsdn = tsdn_fetch();
2747-
witness_assert_lockless(tsdn);
2747+
witness_assert_lock_depth(tsdn, 0);
27482748
stats_print(write_cb, cbopaque, opts);
2749-
witness_assert_lockless(tsdn);
2749+
witness_assert_lock_depth(tsdn, 0);
27502750
}
27512751

27522752
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
@@ -2759,14 +2759,14 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
27592759
malloc_thread_init();
27602760

27612761
tsdn = tsdn_fetch();
2762-
witness_assert_lockless(tsdn);
2762+
witness_assert_lock_depth(tsdn, 0);
27632763

27642764
if (config_ivsalloc)
27652765
ret = ivsalloc(tsdn, ptr, config_prof);
27662766
else
27672767
ret = (ptr == NULL) ? 0 : isalloc(tsdn, ptr, config_prof);
27682768

2769-
witness_assert_lockless(tsdn);
2769+
witness_assert_lock_depth(tsdn, 0);
27702770
return (ret);
27712771
}
27722772

0 commit comments

Comments
 (0)